diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml new file mode 100644 index 000000000..94a4f3bc7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -0,0 +1,34 @@ +name: "Bug report" +description: Bug report template +title: "A concise description" +labels: ["bug", "triage"] +body: + - type: input + id: version + attributes: + label: Version + validations: + required: true + - type: input + id: Evnironment + attributes: + label: Environment + description: (OS, Python version, deployment environment) + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: actual-behavior + attributes: + label: Actual behavior + validations: + required: true + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to reproduce + validations: + required: true \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml new file mode 100644 index 000000000..a73d76781 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -0,0 +1,12 @@ +name: "Feature request" +description: Feature request template +title: "A concise description" +labels: ["feature request"] +body: + - type: textarea + id: desired-behavior + attributes: + label: Feature request description/rationale + validations: + required: true + \ No newline at end of file diff --git a/.github/workflows/python-app-ci.yml b/.github/workflows/python-app-ci.yml index 145adba45..f2d5f17d1 100644 --- a/.github/workflows/python-app-ci.yml +++ b/.github/workflows/python-app-ci.yml @@ -7,7 +7,6 @@ on: push: branches: - 'main' - - 'development' pull_request: workflow_dispatch: @@ -17,10 +16,14 @@ jobs: strategy: fail-fast: false matrix: - os: [ macos-13, ubuntu-latest ] + os: [ macos-latest, ubuntu-latest ] steps: - uses: actions/checkout@v3 + - name: Install libsodium + if: runner.os == 'macOS' + run: | + brew install libsodium - name: Set up Python 3.12.2 uses: actions/setup-python@v2 with: @@ -36,6 +39,10 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --ignore=E7,F841,E301,E302,E303 --max-complexity=10 --max-line-length=127 --statistics + - name: Set DYLD_FALLBACK_LIBRARY_PATH + if: runner.os == 'macOS' + run: | + echo "DYLD_FALLBACK_LIBRARY_PATH=$(brew --prefix libsodium)/lib" >> $GITHUB_ENV - name: Run core KERI tests run: | pytest tests/ --ignore tests/demo/ --ignore test/scripts @@ -60,9 +67,9 @@ jobs: run: | pytest --cov=./ --cov-report=xml - name: Upload - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: - token: 5eb1c60e-8b43-45f1-a003-357b240061cd + token: ${{ secrets.CODECOV_TOKEN }} scripts: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 8e707bc7f..2be298802 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,7 @@ klid.log config/ # keripy virtualenv -keripy/ \ No newline at end of file +keripy/ + +# ctags +.tags diff --git a/.readthedocs.yaml b/.readthedocs.yaml index ceb3023a3..43e7e2fed 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,7 +11,7 @@ build: apt_packages: - libsodium23 tools: - python: "3.10" + python: "3.12.1" # You can also specify other tool versions: # nodejs: "16" # rust: "1.55" diff --git a/Contributing.md b/Contributing.md index ccef9a424..3f59768e9 100644 --- a/Contributing.md +++ b/Contributing.md @@ -81,7 +81,7 @@ Adapted from the Rust Code of Conduct: [https://www.rust-lang.org/policies/code- ## Getting Help For questions or clarifications, reach out via: -- Discord: [https://discord.gg/edGDD632tP](https://discord.gg/edGDD632tP) +- Discord: [See KERI repo] (https://github.com/WebOfTrust/keri) - KERI Development Meetings: [https://github.com/WebOfTrust/keri#implementors-call] - ACDC Standards Meeting@TOIP (technically must be a member of ToIP to contribute): [https://github.com/WebOfTrust/keri#specification-call] @@ -91,4 +91,4 @@ Thanks to all our wonderful contributors. ## Additional Resources - [Project Documentation and Search Engine](https://kerisse.org) -- [Related Projects](https://github.com/WebOfTrust) \ No newline at end of file +- [Related Projects](https://github.com/WebOfTrust) diff --git a/Makefile b/Makefile index 5a4ad3d5e..5af2cce47 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,11 @@ -.PHONY: build-keri +.PHONY: build-keri build-witness-demo publish-keri-witness-demo publish-keri -VERSION=1.1.33 +VERSION=1.2.13 +REGISTRY=weboftrust +IMAGE=keri +LATEST_TAG=$(REGISTRY)/$(IMAGE):latest +VERSIONED_TAG=$(REGISTRY)/$(IMAGE):$(VERSION) define DOCKER_WARNING In order to use the multi-platform build enable the containerd image store @@ -12,19 +16,28 @@ To enable the feature for Docker Desktop: Select Apply and Restart." endef +.PHONY: build-keri build-keri: .warn - @docker build --platform=linux/amd64,linux/arm64 -f images/keripy.dockerfile -t weboftrust/keri:$(VERSION) . + @docker build \ + --platform=linux/amd64,linux/arm64 \ + -f images/keripy.dockerfile \ + -t $(VERSIONED_TAG) \ + -t $(LATEST_TAG) . .PHONY: build-witness-demo build-witness-demo: .warn - @docker build --platform=linux/amd64,linux/arm64 -f images/witness.demo.dockerfile -t weboftrust/keri-witness-demo:1.1.10 . + @docker build \ + --platform=linux/amd64,linux/arm64 \ + -f images/witness.demo.dockerfile \ + -t $(REGISTRY)/keri-witness-demo:$(VERSION) . .PHONY: publish-keri-witness-demo publish-keri-witness-demo: - @docker push weboftrust/keri-witness-demo --all-tags + @docker push $(REGISTRY)/keri-witness-demo --all-tags +.PHONY: publish-keri publish-keri: - @docker push weboftrust/keri:$(VERSION) + @docker push $(REGISTRY)/$(IMAGE) --all-tags .warn: @echo -e ${RED}"$$DOCKER_WARNING"${NO_COLOUR} diff --git a/README.md b/README.md index b281a2f11..a65692358 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ to get a version string similar to the following: ### Local installation - Docker build Run `make build-keri` to build your docker image. -Then run `docker run --pull=never -it --entrypoint /bin/bash weboftrust/keri:1.1.33` and you can run `kli version` from within the running container to play with KERIpy. +Then run `docker run --pull=never -it --entrypoint /bin/bash weboftrust/keri:1.1.10` and you can run `kli version` from within the running container to play with KERIpy. Make sure the image tag matches the version used in the `Makefile`. We use `--pull=never` to ensure that docker does not implicitly pull a remote image and relies on the local image tagged during `make build-keri`. @@ -34,7 +34,7 @@ We use `--pull=never` to ensure that docker does not implicitly pull a remote im ### Dependencies #### Binaries -python 3.10.4+ +python 3.12.1+ libsodium 1.0.18+ @@ -66,7 +66,7 @@ $ pip3 install -U cbor2 ## Development ### Setup -* Ensure Python 3.10.4 is present along with venv and dev header files; +* Ensure Python 3.12.1 is present along with venv and dev header files; * Setup virtual environment: `python3 -m venv keripy` * Activate virtual environment: `source keripy/bin/activate` * Setup dependencies: `pip install -r requirements.txt` @@ -88,3 +88,17 @@ pytest tests/demo/ * Build with Sphinx in `/docs`: * `$ make html` +## Publishing containers + +Enable the containerd image store + +The containerd image store isn't enabled by default. To enable the feature for Docker Desktop: + +Navigate to Settings in Docker Desktop. +In the General tab, check Use containerd for pulling and storing images. +Select Apply & Restart. + +```shell +make build-keri +make publish-keri +``` \ No newline at end of file diff --git a/docs/keri_db.rst b/docs/keri_db.rst index 14327b0fc..e2354b170 100644 --- a/docs/keri_db.rst +++ b/docs/keri_db.rst @@ -13,8 +13,6 @@ keri.db.dbing .. automodule:: keri.db.dbing :members: -The `KERI_LMDB_MAP_SIZE` environment variable can be used to set the size of the LMDB map. The default size is 4GB. - keri.db.escrowing ----------------- diff --git a/docs/naming.md b/docs/naming.md index 8002f29b3..d25809958 100644 --- a/docs/naming.md +++ b/docs/naming.md @@ -117,6 +117,12 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words when namespacing. First ref is module not package variable such as: core.behaving, core.clustering +### Public Module Level Class Instances Singletons + UpperCamelCase + +### Private Module Level Class Instances Singletons + _LeadingUnderscoreUpperCamelCase + ### Public Module Level Methods and Attributes: lowerCamelCase. Methods use verbs. @@ -124,15 +130,15 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words Examples: getName setStyle, display, first, last, itemCount, entities, books, data ### Private Module Level Methods and Attributes (not exported by from import `*`): - leadingUnderscoreLowerCamelCase. + _leadingUnderscoreLowerCamelCase. Methods use verbs. Attributes that are sequences use plural nouns. Examples: `_dirtyBit` ### Constants Module Level: - UPPER_CASE_WITH_UNDERSCORES + UPPER_CASE_WITH_UNDERSCORES or ALLUPPERCASE Not meant to be changed should be numbers or strings. - Examples: MY_CONSTANT + Examples: MY_CONSTANT, SOMECONSTANT ### Dynamic Global Variables (not constants) Module Level: Capitalized_With_Underscores. @@ -158,7 +164,7 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words Example: TotalInstances, Storage, Store, Redact ### Private Class Attributes, Class Methods, Static Methods: - LeadingUnderscoreUpperCamelCase, + _LeadingUnderscoreUpperCamelCase, Methods use verbs. Attributes that are sequences use plural nouns. Example: `_TotalInstances _Storage __Entries` @@ -177,7 +183,7 @@ When using CapCamelCase or mixedCase the acronyms should be treated as words ### Private Instance and Attributes (not exported with from import `*``): - leadingUnderscoreLowerCamelCase. + _leadingUnderscoreLowerCamelCase. Methods use verbs. Attributes that are sequences use plural nouns. Examples: `_getScore _setInternal _lastName _count _entries` @@ -207,7 +213,7 @@ Verticle white space matters. Code should have paragraphs. Balanced brackets, indendation and blank lines demarcate paragraphs For example - +```c void display(void) { int start; @@ -217,13 +223,16 @@ void display(void) if(start == -1) return; } +``` is more readable than +```c void display(void){ int start; start = -1; if(start == -1) return; } +``` Line length matters. Which means variable length matters. Simple logic statments that wrap are no longer simple. @@ -271,44 +280,76 @@ that are sufficiently evocative. ## Suffix Mapping Adjective describing module. What does the module enable one to do. + Verb is the deed or act + Object is actor doer + Place to keep track of or create Objects container or factory + doery actorery doerage actorage + of an Doer is a doery or doage or dodom or dohood -er -or -eur -ster Agent one who does something brewer (Object Classes) + -ery a place for an actor to act factory brewery (Object Factories) + -ing action of running, wishing (Module Names) + -age state of acting actor to act brewerage + -dom state of doing acting kingdom + -hood state of being childhood + -ship quality of or state of rank of midship + -ize to make itemize + -izer Someone who makes one do itemizer + -ive having nature of active rotative inceptive + -acy -isy -ty quality of linty piracy clerisy + -ion -tion -sion act or state of action itemization + -y -ly like full of happening noisy monthly patron + patroner + patronist + patronery + patronage + patrondom + patronship + patronacy + patronhood + patronize + patronish + patronive + patronlet + patronly + patrony + ### Rules for English suffixes http://www.paulnoll.com/Books/Clear-English/English-suffixes-1.html @@ -316,51 +357,97 @@ http://www.prefixsuffix.com/rootchart.php Suffix Meaning Examples Used + able, ible capable of, worthy agreeable, comfortable, credible + age act of or state of salvage, bondage + acy, isy quality hypocrisy, piracy + al, eal, ial on account of related to, action of judicial, official arrival, refusal + ance, ence act or fact of doing state of violence, dependence allowance, insurance + ant quality of one who defiant, expectant, reliant occupant, accountant + er, or, eur agent, one who author, baker, winner, dictator, chauffeur, worker + ed past jumped, baked + ery a place to practice of condition of nunnery, cannery surgery bravery, drudgery + dom state, condition of wisdom, kingdom, martyrdom + ent having the quality of different, dependent, innocent + en made of, to make woolen, wooden, darken -er degree of comparison harder, newer, older 5440 -est highest of comparison cleanest, hardest, softest 1339 -ful full of graceful, restful, faithful 212 -hood state of being boyhood, knighthood, womanhood 61 -ible, ile, il capable of being digestible, responsible, docile, civil 783 -ier, ior one who carrier, warrior 1114 -ify to make magnify, beautify, falsify 125 -ic like, made of metallic, toxic, poetic 3774 -ing action of running, wishing 5440 -ion act or state of confusion, correction, protection 3362 -ism fact of being communism, socialism 1147 -ish like childish, sheepish, foolish 566 -ist a person who does artist, geologist 1375 -ity, ty state of majesty, chastity, humanity 4795 -itis inflammation of appendicitis, tonsillitis 124 -ive having nature of attractive, active 961 -ize to make pasteurize, motorize 637 -less without motionless, careless, childless 282 - -let small starlet, eaglet 185 -ly like, in a manner happening heavenly, remarkably, suddenly every absolutely, monthly 5440 -ment state or quality act of doing accomplishment, excitement placement, movement 680 -meter device for measuring thermometer, barometer 166 -ness state of blindness, kindness 3322 -ology study of geology, zoology, archaeology 374 -ous, ious full of joyous, marvelous, furious 1615 -ship quality of or state of rank of friendship, leadership lordship -scope instrument for seeing telescope, microscope -some like tiresome, lonesome -tion, sion action, state of being condition, attention, fusion -ty quality or state of liberty, majesty -ward toward southward, forward -y like, full of, diminutive: noisy, sooty, kitty -ure noun from verv indicating act or office seizure prefecture +er degree of comparison harder, newer, older + +est highest of comparison cleanest, hardest, softest + +ful full of graceful, restful, faithful + +-hood noun from noun group state of being boyhood, knighthood, womanhood + +ible, ile, il capable of being digestible, responsible, docile, civil + +ier, ior one who carrier, warrior + +ify to make magnify, beautify, falsify + +ic like, made of metallic, toxic, poetic + +ing action of running, wishing + +ion, sion, tion act or state of being confusion, correction, protection + +ism fact of being communism, socialism + +ish like childish, sheepish, foolish + +ist a person who does artist, geologist + +ity, ty state of majesty, chastity, humanity + +itis inflammation of appendicitis, tonsillitis + +ive having nature of attractive, active + +ize to make pasteurize, motorize + +less without motionless, careless, childless + + +let small starlet, eaglet + +ly like, in a manner happening heavenly, remarkably, suddenly every absolutely, monthly + +ment state or quality act of doing accomplishment, excitement placement, movement + +meter device for measuring thermometer, barometer + +ness state of blindness, kindness + +ology study of geology, zoology, archaeology + +ous, ious full of joyous, marvelous, furious + +-ship noun from noun quality of state of rank of friendship, leadership lordship + +-scope instrument for seeing telescope, microscope + +-some like tiresome, lonesome + +-tude noun from adjective state of altitude, latitude + +-ty quality or state of liberty, majesty + +-ward toward southward, forward + +-y like, full of, diminutive: noisy, sooty, kitty + +-ure noun from verb indicating act or office seizure prefecture + + diff --git a/images/keripy.dockerfile b/images/keripy.dockerfile index 71b36b2ec..5591bc801 100644 --- a/images/keripy.dockerfile +++ b/images/keripy.dockerfile @@ -1,4 +1,4 @@ -ARG BASE=python:3.12.2-alpine3.19 +ARG BASE=python:3.12.3-alpine3.20 FROM ${BASE} AS builder diff --git a/images/witness.demo.dockerfile b/images/witness.demo.dockerfile index f19b068ac..f1d554d40 100644 --- a/images/witness.demo.dockerfile +++ b/images/witness.demo.dockerfile @@ -1,4 +1,4 @@ -FROM gleif/keri:1.1.0 +FROM gleif/keri:latest SHELL ["/bin/bash", "-c"] EXPOSE 5632 diff --git a/scripts/demo/README.md b/scripts/demo/README.md index 65572bdf9..dc0a94080 100644 --- a/scripts/demo/README.md +++ b/scripts/demo/README.md @@ -50,16 +50,6 @@ following 2 subcommands are available for starting witnesses: For most of the scripts that require witnesses you will use `kli witness demo` to start the 3 known witnesses. -### Running Agents -Agents can be started in several ways using the `kli agent` subcommands or the shell script `demo/basic/start-agent.sh` for the -scripts that execute `curl` commands against running agents. The following 3 subcommands are available for starting -agents: - -* `kli agent start` - starts a single agent (used inside the start-agent.sh scripts) -* `kli agent demo` - starts 4 agents running on known ports (ports that are expected from many of the scripts) -* `kli agent vlei` - starts 10 agents running on known ports for the vLEI ecosystem scripts. - -Each directory details how to run agents for the scripts it contains and what parameters need to be passed. ### Additional Software Some scripts require addition services from other repositories. Those requirements are listed in the README.md files diff --git a/scripts/demo/basic/challenge.sh b/scripts/demo/basic/challenge.sh index c3fcc6ccb..28d412855 100755 --- a/scripts/demo/basic/challenge.sh +++ b/scripts/demo/basic/challenge.sh @@ -11,6 +11,9 @@ kli ends add --name cha2 --alias cha2 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZE cha1_oobi="$(kli oobi generate --name cha1 --alias cha1 --role witness | sed -n '2 p')" cha2_oobi="$(kli oobi generate --name cha2 --alias cha2 --role witness | sed -n '2 p')" +echo "${cha1_oobi}" +echo "${cha2_oobi}" + kli oobi resolve --name cha1 --oobi-alias cha2 --oobi "${cha2_oobi}" kli oobi resolve --name cha2 --oobi-alias cha1 --oobi "${cha1_oobi}" diff --git a/scripts/demo/basic/clean.sh b/scripts/demo/basic/clean.sh new file mode 100755 index 000000000..2862075d9 --- /dev/null +++ b/scripts/demo/basic/clean.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +echo "Testing clean" + +function isSuccess() { + ret=$? + if [ $ret -ne 0 ]; then + echo "Error $ret" + exit $ret + fi +} + +# CREATE DATABASE AND KEYSTORE +kli init --name nat --base "${KERI_TEMP_DIR}" --nopasscode +isSuccess + +# RESOLVE WITNESS OOBIs +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller +isSuccess +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller +isSuccess +kli oobi resolve --name nat --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller +isSuccess + +## INCEPT AND PROPOGATE EVENTS AND RECEIPTS TO WITNESSES +kli incept --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli rotate --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +isSuccess +kli rotate --name nat --base "${KERI_TEMP_DIR}" --receipt-endpoint --alias nat --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess +kli interact --name nat --base "${KERI_TEMP_DIR}" --alias nat +isSuccess + +kli clean --name nat --base "${KERI_TEMP_DIR}" +isSuccess + +kli status --name nat --alias nat diff --git a/scripts/demo/basic/delegate.sh b/scripts/demo/basic/delegate.sh index cefc2fa3e..9237a5deb 100755 --- a/scripts/demo/basic/delegate.sh +++ b/scripts/demo/basic/delegate.sh @@ -9,7 +9,6 @@ kli incept --name delegate --alias delegate --proxy proxy --file ${KERI_DEMO_SCR pid=$! PID_LIST+=" $pid" -sleep 2 kli delegate confirm --name delegator --alias delegator -Y & pid=$! PID_LIST+=" $pid" @@ -23,10 +22,24 @@ kli rotate --name delegate --alias delegate --proxy proxy & pid=$! PID_LIST="$pid" -sleep 2 echo "Checking for delegate rotate..." kli delegate confirm --name delegator --alias delegator -Y & pid=$! PID_LIST+=" $pid" wait $PID_LIST + +kli status --name delegate --alias delegate + +kli init --name validator --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis --salt 0ACDEyMzQ1Njc4OWxtbm9vAl + +OOBI=$(kli oobi generate --name delegator --alias delegator --role witness | head -n 1) +kli oobi resolve --name validator --oobi-alias delegator --oobi "${OOBI}" +OOBI=$(kli oobi generate --name delegate --alias delegate --role witness | head -n 1) +kli oobi resolve --name validator --oobi-alias delegate --oobi "${OOBI}" + +AID=$(kli aid --name delegate --alias delegate) +kli kevers --name validator --prefix "${AID}" + + + diff --git a/scripts/demo/basic/demo-witness-async-script.sh b/scripts/demo/basic/demo-witness-async-script.sh index 57885a7cb..26b6aac89 100755 --- a/scripts/demo/basic/demo-witness-async-script.sh +++ b/scripts/demo/basic/demo-witness-async-script.sh @@ -13,34 +13,34 @@ function isSuccess() { } # CREATE DATABASE AND KEYSTORE -kli init --name witness-test --base "${KERI_TEMP_DIR}" --nopasscode +kli init --name async-witness-test --base "${KERI_TEMP_DIR}" --nopasscode isSuccess # RESOLVE WITNESS OOBIs -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wan --oobi http://127.0.0.1:5643/oobi/BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM/controller isSuccess -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wil --oobi http://127.0.0.1:5642/oobi/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha/controller isSuccess -kli oobi resolve --name witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller +kli oobi resolve --name async-witness-test --base "${KERI_TEMP_DIR}" --oobi-alias wes --oobi http://127.0.0.1:5644/oobi/BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX/controller isSuccess ## INCEPT AND PROPOGATE EVENTS AND RECEIPTS TO WITNESSES -kli incept --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" +kli incept --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" isSuccess -kli incept --name witness-test --base "${KERI_TEMP_DIR}" --alias inquisitor --file "${KERI_DEMO_SCRIPT_DIR}/data/inquisitor-sample.json" +kli incept --name async-witness-test --base "${KERI_TEMP_DIR}" --alias inquisitor --file "${KERI_DEMO_SCRIPT_DIR}/data/inquisitor-sample.json" isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits -kli rotate --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +kli rotate --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-cut BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits -kli rotate --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX +kli rotate --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits --witness-add BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX isSuccess -kli status --name witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits +kli status --name async-witness-test --base "${KERI_TEMP_DIR}" --alias trans-wits echo 'Test Complete' \ No newline at end of file diff --git a/scripts/demo/basic/essr.sh b/scripts/demo/basic/essr.sh deleted file mode 100755 index e3a5cb957..000000000 --- a/scripts/demo/basic/essr.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -#kli init --name sender --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis -#kli incept --name sender --alias sender --file ${KERI_DEMO_SCRIPT_DIR}/data/gleif-sample.json -# -#kli init --name recipient --salt 0ACDEyMzQ1Njc4OWxtbm9qWc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis -#kli incept --name recipient --alias recipient --file ${KERI_DEMO_SCRIPT_DIR}/data/gleif-sample.json -# -#kli oobi resolve --name sender --oobi-alias recipient --oobi http://127.0.0.1:5642/oobi/EFXJsTFSo10FAZGR-8_Uw1DlhU8nuRFOAN9Z8ajJ56ci/witness -#kli oobi resolve --name recipient --oobi-alias sender --oobi http://127.0.0.1:5642/oobi/EJf7MfzNmehwY5310MUWXPSxhAA_3ifPW2bdsjwqnvae/witness -# -#kli vc registry incept --name sender --alias sender --registry-name vLEI -# -#kli vc create --name sender --alias sender --registry-name vLEI --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao --recipient ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k --data @${KERI_DEMO_SCRIPT_DIR}/data/credential-data.json -#SAID=$(kli vc list --name sender --alias sender --issued --said --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao) -# -#kli ipex grant --name sender --alias sender --said "${SAID}" --recipient ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k - -kli essr send --name sender --alias sender --recipient recipient \ No newline at end of file diff --git a/scripts/demo/basic/multisig-for-revoke-auth.sh b/scripts/demo/basic/multisig-for-revoke-auth.sh new file mode 100755 index 000000000..29599e89d --- /dev/null +++ b/scripts/demo/basic/multisig-for-revoke-auth.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# This script models revokable authority + +# Initialize and incept the 3 parties +kli init -n multisig1 --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis +kli incept -n multisig1 --alias multisig1 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json + +kli init -n multisig2 --salt 0ACDEyMzQ1Njc4OWdoaWphea --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis +kli incept -n multisig2 --alias multisig2 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json + +kli init -n multisig3 --salt 0ACDEyMzQ1Njc4OWdoaWpomw --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis +kli incept -n multisig3 --alias multisig3 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-3-sample.json + +# Resolve OOBIs to establish connections +kli oobi resolve -n multisig1 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC/witness +kli oobi resolve -n multisig2 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EI0jXuw_V_zjj_mFgJLJWgFtbpVRNdUmv01WoM4na1ek/witness + +kli oobi resolve -n multisig1 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC/witness +kli oobi resolve -n multisig3 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EI0jXuw_V_zjj_mFgJLJWgFtbpVRNdUmv01WoM4na1ek/witness + +kli oobi resolve -n multisig2 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC/witness +kli oobi resolve -n multisig3 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC/witness + +# Incept a multisig group for multisig1 and multisig2 using a shared configuration file +kli multisig incept -n multisig1 --alias multisig1 --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-sign-auth.json & +pid=$! +PID_LIST+=" $pid" + +kli multisig incept -n multisig2 --alias multisig2 --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-sign-auth.json & +pid=$! +PID_LIST+=" $pid" + +wait $PID_LIST + +# Rotate the keys for multisig2 +# This models the ability of the second party to rotate keys and potentially exclude the first party's key +kli rotate -n multisig2 --alias multisig2 + +# Query the state of multisig3 to check the current configuration +kli query --name multisig3 --alias multisig3 --prefix EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC + +# Resolve OOBIs for multisig3 to update its state with the latest information +kli oobi resolve -n multisig3 --oobi-alias multisig --oobi http://127.0.0.1:5642/oobi/EPKgQWXeBFsE9DjyqvspoPX1JVmRbRlNkCCaqvEeppM6/witness + +# Perform a multisig rotate operation for multisig2 +# The smids (signing member identifiers) and rmids (rotation member identifiers) are used to configure the new state +# The new state excludes the first party's key, effectively revoking its signing privileges +kli multisig rotate -n multisig2 --alias multisig \ + --smids EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC:1 \ + --smids EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC:0 \ + --isith '["0","1"]' \ + --rmids EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC:1 \ + --rmids EIwtBwakOchYfReVjZnou_ZR9pA9Sjd877Y4pegfOGSC:0 \ + --nsith '["1","0"]' & +pid=$! +PID_LIST="$pid" + +# Join the multisig group for multisig3 to synchronize its state +kli multisig join --name multisig3 --auto & +pid=$! +PID_LIST+=" $pid" + +# Wait for all background processes to complete +wait $PID_LIST \ No newline at end of file diff --git a/scripts/demo/basic/multisig-rotate-three-stooges.sh b/scripts/demo/basic/multisig-rotate-three-stooges.sh new file mode 100755 index 000000000..5db39bfa6 --- /dev/null +++ b/scripts/demo/basic/multisig-rotate-three-stooges.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# three stooges + +source ${KERI_SCRIPT_DIR}/demo/basic/script-utils.sh + +# WITNESSES +# To run the following scripts, open another console window and run: +# $ kli witness demo + +print_yellow "Multisig rotation with three AIDs" +echo + +kli init --name larry --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +# Prefix EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U +kli incept --name larry --alias larry --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json + +kli init --name moe --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +# Prefix ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW +kli incept --name moe --alias moe --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json + +kli init --name curly --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +# Prefix EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu +kli incept --name curly --alias curly --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-3-sample.json + +# OOBI resolution does the initial discovery of key state +echo +print_yellow "Resolve OOBIs" +kli oobi resolve --name larry --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name larry --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + +kli oobi resolve --name moe --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name moe --oobi-alias curly --oobi http://127.0.0.1:5642/oobi/EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + +kli oobi resolve --name curly --oobi-alias larry --oobi http://127.0.0.1:5642/oobi/EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name curly --oobi-alias moe --oobi http://127.0.0.1:5642/oobi/ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + +# Multisig Inception +echo +print_yellow "Multisig Inception" +# Follow commands run in parallel +kli multisig incept --name larry --alias larry --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-three-aids.json & +pid=$! +PID_LIST+=" $pid" +kli multisig incept --name moe --alias moe --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-three-aids.json & +pid=$! +PID_LIST+=" $pid" +kli multisig incept --name curly --alias curly --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-three-aids.json & +pid=$! +PID_LIST+=" $pid" + +echo +print_yellow "Multisig Inception - wait" +wait $PID_LIST + +echo +print_green "Multisig Inception - status" +kli status --name larry --alias multisig + +# Rotate keys for each multisig - required before rotating the multisig +echo +print_yellow "Rotate keys for each multisig" +kli rotate --name larry --alias larry +kli rotate --name moe --alias moe +kli rotate --name curly --alias curly + +# Pull key state in from other multisig group participant identifiers so they have the next digest +echo +print_yellow "Pull key state in from other multisig group participant identifiers" +# 2 about 1 +kli query --name moe --alias moe --prefix EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U +# 2 about 3 +kli query --name moe --alias moe --prefix EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu +# 1 about 2 +kli query --name larry --alias larry --prefix ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW +# 1 about 3 +kli query --name larry --alias larry --prefix EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu +# 3 about 1 +kli query --name curly --alias curly --prefix EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U +# 3 about 2 +kli query --name curly --alias curly --prefix ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW + + +echo +print_yellow "Multisig rotation" + +PID_LIST="" + +kli multisig rotate --name larry --alias multisig --smids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --smids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --smids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --rmids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --rmids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu & +pid=$! +PID_LIST+=" $pid" +kli multisig rotate --name moe --alias multisig --smids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --smids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --smids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --rmids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --rmids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu & +pid=$! +PID_LIST+=" $pid" +kli multisig rotate --name curly --alias multisig --smids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --smids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --smids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U --rmids ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW --rmids EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu & +pid=$! +PID_LIST+=" $pid" + +echo +print_yellow "Multisig rotation - wait" +wait $PID_LIST + +echo +print_green "Multisig rotation - status" +kli status --name larry --alias multisig + +echo +print_yellow "Multisig interact" + +PID_LIST="" + +kli multisig interact --name larry --alias multisig --data "{\"tagline\":\"three lost souls\"}" & +pid=$! +PID_LIST+=" $pid" +kli multisig interact --name moe --alias multisig --data "{\"tagline\":\"three lost souls\"}" & +pid=$! +PID_LIST+=" $pid" +kli multisig interact --name curly --alias multisig --data "{\"tagline\":\"three lost souls\"}" & +pid=$! +PID_LIST+=" $pid" + +echo +print_yellow "Multisig interact - wait" +wait $PID_LIST + +echo +print_green "Multisig interact - status" +kli status --name larry --alias multisig +print_lcyan "Multisig rotate three stooges - done." diff --git a/scripts/demo/basic/multisig-rotation-in-third.sh b/scripts/demo/basic/multisig-rotation-in-third.sh index 55d384a43..79f3b5382 100755 --- a/scripts/demo/basic/multisig-rotation-in-third.sh +++ b/scripts/demo/basic/multisig-rotation-in-third.sh @@ -17,8 +17,6 @@ kli oobi resolve --name multisig1 --oobi-alias multisig2 --oobi http://127.0.0.1 kli oobi resolve --name multisig1 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha kli oobi resolve --name multisig2 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha kli oobi resolve --name multisig2 --oobi-alias multisig3 --oobi http://127.0.0.1:5642/oobi/ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name multisig3 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha -kli oobi resolve --name multisig3 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha # Follow commands run in parallel kli multisig incept --name multisig1 --alias multisig1 --group multisig --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-sample.json & @@ -34,6 +32,13 @@ kli status --name multisig1 --alias multisig PID_LIST="" +kli rotate --name multisig1 --alias multisig1 +kli query --name multisig2 --alias multisig2 --prefix EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 +kli rotate --name multisig2 --alias multisig2 +kli query --name multisig1 --alias multisig1 --prefix EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 +kli oobi resolve --name multisig3 --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha +kli oobi resolve --name multisig3 --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1/witness/BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha + kli multisig rotate --name multisig1 --alias multisig --smids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --smids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --smids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U --isith '["1/3", "1/3", "1/3"]' --nsith '["1/2", "1/2", "1/2"]' --rmids EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 --rmids EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 --rmids ENkjt7khEI5edCMw5qugagbJw1QvGnQEtcewxb0FnU9U & pid=$! PID_LIST+=" $pid" diff --git a/scripts/demo/basic/multisig.sh b/scripts/demo/basic/multisig.sh index e6c6a8000..78006672d 100755 --- a/scripts/demo/basic/multisig.sh +++ b/scripts/demo/basic/multisig.sh @@ -6,11 +6,11 @@ kli init --name multisig1 --base "${KERI_TEMP_DIR}" --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis kli incept --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-1-sample.json -kli ends add --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM --role mailbox +#kli ends add --name multisig1 --base "${KERI_TEMP_DIR}" --alias multisig1 --eid BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM --role mailbox kli init --name multisig2 --base "${KERI_TEMP_DIR}" --salt 0ACDEyMzQ1Njc4OWdoaWpsaw --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis kli incept --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --file ${KERI_DEMO_SCRIPT_DIR}/data/multisig-2-sample.json -kli ends add --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --eid BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX --role mailbox +#kli ends add --name multisig2 --base "${KERI_TEMP_DIR}" --alias multisig2 --eid BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX --role mailbox kli oobi resolve --name multisig1 --base "${KERI_TEMP_DIR}" --oobi-alias multisig2 --oobi http://127.0.0.1:5642/oobi/EJccSRTfXYF6wrUVuenAIHzwcx3hJugeiJsEKmndi5q1 kli oobi resolve --name multisig2 --base "${KERI_TEMP_DIR}" --oobi-alias multisig1 --oobi http://127.0.0.1:5642/oobi/EKYLUMmNPZeEs77Zvclf0bSN5IN-mLfLpx2ySb-HDlk4 diff --git a/scripts/demo/basic/rename-alias.sh b/scripts/demo/basic/rename-alias.sh new file mode 100755 index 000000000..5867aa174 --- /dev/null +++ b/scripts/demo/basic/rename-alias.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# CREATE DATABASE AND KEYSTORE +kli init --name rename-test --base "${KERI_TEMP_DIR}" --nopasscode + +# Incept with the initial alias "sabbir" +kli incept --name rename-test --base "${KERI_TEMP_DIR}" --alias sabbir --file ${KERI_DEMO_SCRIPT_DIR}/data/transferable-sample.json + +# Rename the alias from "sabbir" to "irfan" +kli rename --name rename-test --base "${KERI_TEMP_DIR}" --alias sabbir irfan + +# Extract alias from status +irfan_alias=$(kli status --name rename-test --base "${KERI_TEMP_DIR}" --alias irfan | grep -Eo 'Alias:\s+(.+)' | awk '{print $2}') + + + +# Check if the extracted alias is "irfan" +if [ "$irfan_alias" = "irfan" ]; then + echo "Alias successfully changed to 'irfan'." +else + echo "Alias did not change !" + +fi + +echo 'Test Complete' \ No newline at end of file diff --git a/scripts/demo/basic/script-utils.sh b/scripts/demo/basic/script-utils.sh new file mode 100644 index 000000000..f0585afdc --- /dev/null +++ b/scripts/demo/basic/script-utils.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Utility functions +print_green() { + text=$1 + printf "\e[32m${text}\e[0m\n" +} + +print_yellow(){ + text=$1 + printf "\e[33m${text}\e[0m\n" +} + +print_red() { + text=$1 + printf "\e[31m${text}\e[0m\n" +} + +print_lcyan() { + text=$1 + printf "\e[96m${text}\e[0m\n" +} \ No newline at end of file diff --git a/scripts/demo/contacts/demo_contacts.sh b/scripts/demo/contacts/demo_contacts.sh new file mode 100755 index 000000000..f2741662b --- /dev/null +++ b/scripts/demo/contacts/demo_contacts.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Demo script for KLI contact management commands + + +KS_ALICE="contact-demo-alice" +KS_BOB="contact-demo-bob" +SALT_ALICE="0AAQmsjh-C7kAJZQEzdrzwB7" +SALT_BOB="0ABBnsjh-D8lBKZRFzesx1C8" + +kli init --name ${KS_ALICE} --nopasscode --salt ${SALT_ALICE} \ + --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +kli incept --name ${KS_ALICE} --alias alice \ + --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" + +echo "Creating Bob's keystore and identifier..." +kli init --name ${KS_BOB} --nopasscode --salt ${SALT_BOB} \ + --config-dir "${KERI_SCRIPT_DIR}" --config-file demo-witness-oobis +kli incept --name ${KS_BOB} --alias bob \ + --file "${KERI_DEMO_SCRIPT_DIR}/data/trans-wits-sample.json" + +ALICE_OOBI=$(kli oobi generate --name ${KS_ALICE} --alias alice --role witness | head -1) +BOB_OOBI=$(kli oobi generate --name ${KS_BOB} --alias bob --role witness | head -1) +echo "Alice OOBI: ${ALICE_OOBI}" +echo "Bob OOBI: ${BOB_OOBI}" + +echo "Alice adds Bob as a contact..." +kli contacts add --name ${KS_ALICE} \ + --oobi "${BOB_OOBI}" \ + --alias bob \ + --field company=ACME \ + --field role=Engineer + +echo "Bob adds Alice as a contact..." +kli contacts add --name ${KS_BOB} \ + --oobi "${ALICE_OOBI}" \ + --alias alice \ + --field company=GLEIF + +echo "Alice's contacts:" +kli contacts list --name ${KS_ALICE} + +echo "Alice gets Bob by alias..." +kli contacts get --name ${KS_ALICE} --alias bob + +echo "Alice renames 'bob' to 'robert'..." +kli contacts rename --name ${KS_ALICE} \ + --old-alias bob \ + --alias robert +kli contacts get --name ${KS_ALICE} --alias robert + +echo "Alice queries robert's latest key state..." +kli contacts query --name ${KS_ALICE} --alias alice --contact-alias robert + +echo "Alice deletes contact 'robert'..." +kli contacts delete --name ${KS_ALICE} --alias robert --yes + +echo "Alice's contacts after delete:" +kli contacts list --name ${KS_ALICE} || echo "(no contacts)" + +echo "Done." diff --git a/scripts/demo/data/multisig-sign-auth.json b/scripts/demo/data/multisig-sign-auth.json new file mode 100644 index 000000000..ebcc168de --- /dev/null +++ b/scripts/demo/data/multisig-sign-auth.json @@ -0,0 +1,16 @@ +{ + "aids": [ + "EI0jXuw_V_zjj_mFgJLJWgFtbpVRNdUmv01WoM4na1ek", + "EDC4X7ia6uAGGLQ20UgUdcIix_YgWlkNK_wC8e3ShTAC" + ], + "transferable": true, + "wits": [ + "BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha", + "BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM", + "BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX" + ], + "toad": 2, + "isith": ["1", "0"], + "nsith": ["0", "1"] + } + \ No newline at end of file diff --git a/scripts/demo/data/multisig-three-aids.json b/scripts/demo/data/multisig-three-aids.json new file mode 100644 index 000000000..c17b8b2a7 --- /dev/null +++ b/scripts/demo/data/multisig-three-aids.json @@ -0,0 +1,16 @@ +{ + "aids": [ + "EA5g3RMwkjcr_M4fI3k2ShCYlQMpgk3HD9mHhx7ZJs4U", + "ED7yk9oUIe5qRh8ILfTuT_sNHidrxwJ9Bl-tLPoAXbqW", + "EEHyoLseuHa0nuhDj9tBv6N6nU1PILwv4jTt5x8A8uLu" + ], + "transferable": true, + "wits": [ + "BBilc4-L3tFUnfM_wJr4S4OJanAv_VmF_dJNN6vkf2Ha", + "BLskRTInXnMxWaGqcpSyMgo0nYbalW99cGZESrz3zapM", + "BIKKuvBwpmDVA4Ds-EpL5bt9OqPzWPja2LigFYZN2YfX" + ], + "toad": 3, + "isith": "3", + "nsith": "3" +} diff --git a/scripts/demo/test_scripts.sh b/scripts/demo/test_scripts.sh index 7138d6bd8..6cce74d38 100755 --- a/scripts/demo/test_scripts.sh +++ b/scripts/demo/test_scripts.sh @@ -4,6 +4,7 @@ script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source "${script_dir}"/demo-scripts.sh +source "${script_dir}"/basic/script-utils.sh # Launch witnesses in background kli witness demo & @@ -53,11 +54,17 @@ printf "\n************************************\n" isSuccess printf "\n************************************\n" -printf "Running multisig-delegate-delegator.sh" +printf "Running multisig-rotate-three-stooges.sh" printf "\n************************************\n" -"${script_dir}/basic/multisig-delegate-delegator.sh" +"${script_dir}/basic/multisig-rotate-three-stooges.sh" isSuccess +printf "\n************************************\n" +printf "Skipping multisig-delegate-delegator.sh" +printf "\n************************************\n" +#"${script_dir}/basic/multisig-delegate-delegator.sh" +#isSuccess + printf "\n************************************\n" printf "Running challenge.sh" printf "\n************************************\n" @@ -69,3 +76,9 @@ printf "Running multisig-join.sh" printf "\n************************************\n" "${script_dir}/basic/multisig-join.sh" isSuccess + +printf "\n************************************\n" +printf "Running rename.sh" +printf "\n************************************\n" +"${script_dir}/basic/rename-alias.sh" +isSuccess \ No newline at end of file diff --git a/setup.py b/setup.py index a02bc1532..9ab5c4f21 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,12 @@ $ twine upload dist/keri-0.0.1.tar.gz Create release git: +$ git tag # lists all tags +$ git tag -a v0.6.11 -m "new feature" +$ git show v0.6.11 +$ git push --tags # pushes tags to default remote +$ git push wot --tags # pushes tags to wot remote + $ git tag -a v0.4.2 -m "bump version" $ git push --tags $ git checkout -b release_0.4.2 @@ -31,7 +37,7 @@ from setuptools import find_packages, setup setup( name='keri', - version='1.1.33', # also change in src/keri/__init__.py + version='1.2.13', # also change in src/keri/__init__.py license='Apache Software License 2.0', description='Key Event Receipt Infrastructure', long_description="KERI Decentralized Key Management Infrastructure", @@ -70,25 +76,26 @@ ], python_requires='>=3.12.2', install_requires=[ - 'lmdb>=1.4.1', - 'pysodium>=0.7.17', - 'blake3>=0.4.1', - 'msgpack>=1.0.8', - 'cbor2>=5.6.2', - 'multidict>=6.0.5', - 'ordered-set>=4.1.0', - 'hio==0.6.14', - 'multicommand>=1.0.0', - 'jsonschema>=4.21.1', - 'falcon>=3.1.3', - 'hjson>=3.1.0', - 'PyYaml>=6.0.1', - 'apispec>=6.6.0', - 'mnemonic>=0.21', - 'PrettyTable>=3.10.0', - 'http_sfv>=0.9.9', - 'cryptography>=42.0.5', - 'semver>=3.0.2' + 'lmdb>=1.4.1', + 'pysodium>=0.7.17', + 'blake3>=0.4.1', + 'msgpack>=1.0.8', + 'cbor2>=5.6.2', + 'multidict>=6.0.5', + 'ordered-set>=4.1.0', + 'hio>=0.6.14', + 'multicommand==1.0.0', + 'jsonschema>=4.21.1', + 'falcon>=3.1.3', + 'hjson>=3.1.0', + 'PyYaml>=6.0.1', + 'apispec>=6.6.0', + 'mnemonic>=0.21', + 'PrettyTable>=3.10.0', + 'http_sfv>=0.9.9', + 'cryptography>=42.0.5', + 'semver>=3.0.2', + 'qrcode>=7.4.2' ], extras_require={ }, diff --git a/src/keri/__init__.py b/src/keri/__init__.py index 55a0b4264..c1824b0e3 100644 --- a/src/keri/__init__.py +++ b/src/keri/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- -__version__ = '1.1.33' # also change in setup.py +__version__ = '1.2.13' # also change in setup.py diff --git a/src/keri/app/agenting.py b/src/keri/app/agenting.py index 927b4f13f..8b5f7bf59 100644 --- a/src/keri/app/agenting.py +++ b/src/keri/app/agenting.py @@ -14,23 +14,33 @@ from socket import gaierror -from . import httping, forwarding +from . import httping, forwarding, tocking from .. import help from .. import kering -from ..core import eventing, parsing, coring, serdering -from ..core.coring import CtrDex +from .. import core +from ..core import eventing, parsing, coring, serdering, indexing from ..db import dbing from ..kering import Roles -from ..help import helping -import datetime - logger = help.ogler.getLogger() class Receiptor(doing.DoDoer): + """DoDoer for synchronous witness receipting and receipt queries over HTTP. + + Drains `msgs` for witness receipt submissions and `gets` for receipt + queries, pushing processed request dicts to `cues`. + """ def __init__(self, hby, msgs=None, gets=None, cues=None): + """Initialize with shared queues and an HTTP client. + + Parameters: + hby (Habery): Habitat environment for identifier lookups. + msgs (Deck): receipt requests with `pre`, optional `sn`, optional `auths`. + gets (Deck): receipt query requests with `pre`, optional `sn`. + cues (Deck): completed request cues. + """ self.msgs = msgs if msgs is not None else decking.Deck() self.gets = gets if gets is not None else decking.Deck() @@ -42,21 +52,21 @@ def __init__(self, hby, msgs=None, gets=None, cues=None): super(Receiptor, self).__init__(doers=doers) - def receipt(self, pre, sn=None): - """ Returns a generator for witness receipting - - The returns a generator that will submit the designated event to witnesses for receipts using - the synchronous witness API, the propogate the receipts to each of the other witnesses. + def receipt(self, pre, sn=None, auths=None): + """Yield while witnessing an event and returning witness receipt couples. + Sends the event to all witnesses, optionally catches new witnesses up, + then propagates the collected non-transferable receipts to each witness. Parameters: - pre (str): qualified base64 identifier to gather receipts for - sn: (Optiona[int]): sequence number of event to gather receipts for, latest is used if not provided + pre (str): qb64 identifier to receipt for. + sn (int | None): sequence number, defaults to latest. + auths (dict | None): optional map of wit AID to 2FA auth header value. Returns: - list: identifiers of witnesses that returned receipts. - + iterable: witness identifiers that returned receipts. """ + auths = auths if auths is not None else dict() if pre not in self.hby.prefixes: raise kering.MissingEntryError(f"{pre} not a valid AID") @@ -71,7 +81,7 @@ def receipt(self, pre, sn=None): ser = serdering.SerderKERI(raw=msg) # If we are a rotation event, may need to catch new witnesses up to current key state - if ser.ked['t'] in (coring.Ilks.rot,): + if ser.ked['t'] in (coring.Ilks.rot, coring.Ilks.drt,): adds = ser.ked["ba"] for wit in adds: yield from self.catchup(ser.pre, wit) @@ -89,28 +99,29 @@ def receipt(self, pre, sn=None): rcts = dict() for wit, client in clients.items(): - httping.streamCESRRequests(client=client, dest=wit, ims=bytearray(msg), path="/receipts") - end = helping.nowUTC() + datetime.timedelta(seconds=10) - while not client.responses and helping.nowUTC() < end: + headers = dict() + if wit in auths: + headers["Authorization"] = auths[wit] + + httping.streamCESRRequests(client=client, dest=wit, ims=bytearray(msg), path="receipts", headers=headers) + while not client.responses: yield self.tock + rep = client.respond() - if rep is not None and rep.status == 200: + if rep.status == 200: rct = bytearray(rep.body) hab.psr.parseOne(bytearray(rct)) rserder = serdering.SerderKERI(raw=rct) del rct[:rserder.size] # pull off the count code - coring.Counter(qb64b=rct, strip=True) + core.Counter(qb64b=rct, strip=True, gvrsn=kering.Vrsn_1_0) rcts[wit] = rct else: - if rep is not None: - logger.error(f"invalid response {rep.status} from witnesses {wit}") - else: - logger.error(f"no response from witness {wit}") + print(f"invalid response {rep.status} from witnesses {wit}") - for wit in rcts.keys(): - ewits = [w for w in rcts.keys() if w != wit] + for wit in rcts: + ewits = [w for w in rcts if w != wit] wigs = [sig for w, sig in rcts.items() if w != wit] msg = bytearray() @@ -124,7 +135,8 @@ def receipt(self, pre, sn=None): sn=sn, said=ser.said) msg.extend(rserder.raw) - msg.extend(coring.Counter(code=CtrDex.NonTransReceiptCouples, count=len(wigs)).qb64b) + msg.extend(core.Counter(core.Codens.NonTransReceiptCouples, + count=len(wigs), gvrsn=kering.Vrsn_1_0).qb64b) for wig in wigs: msg.extend(wig) @@ -139,18 +151,13 @@ def receipt(self, pre, sn=None): return rcts.keys() def get(self, pre, sn=None): - """ Returns a generator for witness querying + """Yield while querying a witness for receipts of an event identified by pre and sn. - The returns a generator that will request receipts for event identified by pre and sn - - - Parameters: - pre (str): qualified base64 identifier to gather receipts for - sn: (Optiona[int]): sequence number of event to gather receipts for, latest is used if not provided + Picks one witness and issues GET /receipts?pre=&sn=. Parses any + returned receipts into the local parser. Returns: - list: identifiers of witnesses that returned receipts. - + bool: True if the witness returned HTTP 200. """ if pre not in self.hby.prefixes: raise kering.MissingEntryError(f"{pre} not a valid AID") @@ -183,12 +190,13 @@ def get(self, pre, sn=None): return rep.status == 200 def catchup(self, pre, wit): - """ When adding a new Witness, use this method to catch the witness up to the current state of the KEL + """ + Yield while replaying the full KEL for `pre` to the witness. + When adding a new Witness, use this method to catch the witness up to the current state of the KEL Parameters: pre (str): qualified base64 AID of the KEL to send wit (str): qualified base64 AID of the witness to send the KEL to - """ if pre not in self.hby.prefixes: raise kering.MissingEntryError(f"{pre} not a valid AID") @@ -205,19 +213,8 @@ def catchup(self, pre, wit): self.remove([clientDoer]) - def witDo(self, tymth=None, tock=0.0): - """ - Returns doifiable Doist compatibile generator method (doer dog) to process - .kevery and .tevery escrows. - - Parameters: - tymth (function): injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock (float): injected initial tock value - - Usage: - add result of doify on this method to doers list - """ + def witDo(self, tymth=None, tock=0.0, **kwa): + """Doer loop that drains `msgs` and runs witness receipt flow.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -227,25 +224,15 @@ def witDo(self, tymth=None, tock=0.0): msg = self.msgs.popleft() pre = msg["pre"] sn = msg["sn"] if "sn" in msg else None + auths = msg["auths"] if "auths" in msg else None - yield from self.receipt(pre, sn) + yield from self.receipt(pre, sn, auths) self.cues.push(msg) yield self.tock - def gitDo(self, tymth=None, tock=0.0): - """ - Returns doifiable Doist compatibile generator method (doer dog) to process - .kevery and .tevery escrows. - - Parameters: - tymth (function): injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock (float): injected initial tock value - - Usage: - add result of doify on this method to doers list - """ + def gitDo(self, tymth=None, tock=0.0, **kwa): + """Doer loop that drains `gets` and runs witness receipt queries.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -262,51 +249,47 @@ def gitDo(self, tymth=None, tock=0.0): class WitnessReceiptor(doing.DoDoer): - """ - Sends messages to all current witnesses of given identifier (from hab) and waits - for receipts from each of those witnesses and propagates those receipts to each - of the other witnesses after receiving the complete set. + """Witness receipt doer that sends events and propagates receipts. - Removes all Doers and exits as Done once all witnesses have been sent the entire - receipt set. Could be enhanced to have a `once` method that runs once and cleans up - and an `all` method that runs and waits for more messages to receipt. + Uses messenger doers to asynchronously send the event to each witness, + waits for receipts to arrive in `hab.db` (via mailbox processing), then + propagates the full receipt set across the witness group. + Could be enhanced to have a `once` method that runs once and cleans up + and an `all` method that runs and waits for more messages to receipt. """ - def __init__(self, hby, msgs=None, cues=None, force=False, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + def __init__(self, hby, msgs=None, cues=None, force=False, auths=None, **kwa): + """Initialize with queues and optional auth for witness endpoints. Parameters: - hby (Habery): Habitat of the identifier to receipt witnesses - msgs (Deck): incoming messages to publish to witnesses - cues (Deck): outgoing cues of successful messages - force (bool): True means to send witnesses all receipts even if we have a full compliment. - + hby (Habery): Habitat environment for identifier lookups. + msgs (Deck): receipt requests with `pre` and optional `sn`. + cues (Deck): completed request cues. + force (bool): resend receipts even if already complete. + auths (dict | None): optional map of wit AID to auth header value. """ self.hby = hby self.force = force self.msgs = msgs if msgs is not None else decking.Deck() self.cues = cues if cues is not None else decking.Deck() + self.auths = auths if auths is not None else dict() super(WitnessReceiptor, self).__init__(doers=[doing.doify(self.receiptDo)], **kwa) - def receiptDo(self, tymth=None, tock=0.0): - """ - Returns doifiable Doist compatible generator method (doer dog) + def receiptDo(self, tymth=None, tock=None, **kwa): + """Doer loop that sends events to witnesses and propagates receipts. - Usage: - add result of doify on this method to doers list - - Parameters: - tymth is injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock is injected initial tock value + Asynchronously processes witness receipt requests from self.msgs queue. + Sends any required delegation context, replays KEL for new witnesses, + posts the event, waits for receipts to be stored in `hab.db`, then + shares the full receipt set across witnesses. If `force` is false and + all receipts already exist, it skips resubmission. + Pushes the original request to self.cues to signal completion """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessReceiptorTock _ = (yield self.tock) while True: @@ -332,7 +315,8 @@ def receiptDo(self, tymth=None, tock=0.0): witers = [] for wit in wits: - witer = messenger(hab, wit) + auth = self.auths[wit] if wit in self.auths else None + witer = messenger(hab, wit, auth=auth) witers.append(witer) self.extend([witer]) @@ -347,26 +331,26 @@ def receiptDo(self, tymth=None, tock=0.0): witer.msgs.append(bytearray(dmsg)) if ser.ked['t'] in (coring.Ilks.icp, coring.Ilks.dip) or \ - "ba" in ser.ked and wit in ser.ked["ba"]: # Newly added witness, must send full KEL to catch up + "ba" in ser.ked and wit in ser.ked["ba"]: # Newly added witness, must catch up for fmsg in hab.db.clonePreIter(pre=pre): witer.msgs.append(bytearray(fmsg)) witer.msgs.append(bytearray(msg)) # make a copy _ = (yield self.tock) - while True: + while True: # wait for all receipts to arrive wigs = hab.db.getWigs(dgkey) if len(wigs) == len(wits): break _ = yield self.tock - # If we started with all our recipts, exit unless told to force resubmit of all receipts + # If we started with all our receipts, exit unless told to force resubmit of all receipts if completed and not self.force: self.cues.push(evt) continue # generate all rct msgs to send to all witnesses - awigers = [coring.Siger(qb64b=bytes(wig)) for wig in wigs] + awigers = [indexing.Siger(qb64b=bytes(wig)) for wig in wigs] # make sure all witnesses have fully receipted KERL and know about each other for witer in witers: @@ -422,21 +406,21 @@ class WitnessInquisitor(doing.DoDoer): for receipts from each of those witnesses and propagates those receipts to each of the other witnesses after receiving the complete set. - Removes all Doers and exits as Done once all witnesses have been sent the entire - receipt set. Could be enhanced to have a `once` method that runs once and cleans up - and an `all` method that runs and waits for more messages to receipt. + Builds and sends qry/tel queries, pushing the raw sent message to `sent`. + The response parsing happens elsewhere (e.g. mailbox or HTTP response handlers). + Could be enhanced to have a `once` method that runs once and cleans up + and an `all` method that runs and waits for more messages to receipt. """ def __init__(self, hby, reger=None, msgs=None, klas=None, **kwa): - """ - For all msgs, select a random witness from Habitat's current set of witnesses - send the msg and process all responses (KEL replays, RCTs, etc) + """Initialize with a message queue and optional messenger class. Parameters: - hby (Habitat): Habitat of the identifier to use to identify witnesses - msgs: is the message buffer to process and send to one random witness. - + hby (Habery): Habitat environment for endpoint and kever lookup. + reger (Reger | None): optional registry database handle. + msgs (Deck): query requests built by `query`/`telquery`. + klas (type | None): messenger class, defaults to `HTTPMessenger`. """ self.hby = hby self.reger = reger @@ -446,15 +430,16 @@ def __init__(self, hby, reger=None, msgs=None, klas=None, **kwa): super(WitnessInquisitor, self).__init__(doers=[doing.doify(self.msgDo)], **kwa) - def msgDo(self, tymth=None, tock=1.0, **opts): + def msgDo(self, tymth=None, tock=None, **opts): """ - Returns doifiable Doist compatible generator method (doer dog) + Doer loop that sends one query to one selected endpoint. - Usage: - add result of doify on this method to doers list + For all msgs, select a random witness from Habitat's current set of witnesses + send the msg and process all responses (KEL replays, RCTs, etc) + Pushes the raw sent message to self.sent to signal completion. """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessInquisitorTock _ = (yield self.tock) while True: @@ -521,23 +506,20 @@ def msgDo(self, tymth=None, tock=1.0, **opts): yield self.tock - def query(self, pre, r="logs", sn='0', src=None, hab=None, anchor=None, wits=None, **kwa): - """ Create, sign and return a `qry` message against the attester for the prefix + def query(self, pre, r="logs", sn='0', fn='0', src=None, hab=None, anchor=None, wits=None, **kwa): + """Create, sign, and queue a `qry` KEL query request against the attester for the prefix for later sending. Parameters: - src (str): qb64 identifier prefix of source of query - hab (Hab): Hab to use instead of src if provided - pre (str): qb64 identifier prefix being queried for - r (str): query route - sn (str): optional specific hex str of sequence number to query for - anchor (Seal): anchored Seal to search for - wits (list) witnesses to query - - Returns: - bytearray: signed query event - - """ - qry = dict(s=sn) + pre (str): qb64 identifier being queried. + r (str): query route (e.g. "logs"). + sn (str): optional hex sequence number to query for. + fn (str): optional hex start sequence number. + src (str | None): qb64 source identifier (ignored if `hab` provided). + hab (Hab | None): habitat used to sign and route the query. + anchor (Seal | None): anchored seal to search for. + wits (list | None): explicit witnesses to target; otherwise uses endpoints. + """ + qry = dict(s=sn, fn=fn) if anchor is not None: qry["a"] = anchor @@ -548,6 +530,7 @@ def query(self, pre, r="logs", sn='0', src=None, hab=None, anchor=None, wits=Non self.msgs.append(msg) def telquery(self, ri, src=None, i=None, r="tels", hab=None, pre=None, wits=None, **kwa): + """Queue a TEL query request for later sending.""" qry = dict(ri=ri) msg = dict(src=src, pre=pre, target=i, r=r, wits=wits, q=qry) if hab is not None: @@ -557,45 +540,39 @@ def telquery(self, ri, src=None, i=None, r="tels", hab=None, pre=None, wits=None class WitnessPublisher(doing.DoDoer): - """ - Sends messages to all current witnesses of given identifier (from hab) and exits. + """DoDoer that publishes messages to all witnesses for an identifier. - Removes all Doers and exits as Done once all witnesses have been sent the message. Could be enhanced to have a `once` method that runs once and cleans up and an `all` method that runs and waits for more messages to receipt. - """ def __init__(self, hby, msgs=None, cues=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize with publish queue and completion cues. Parameters: - hby (Habery): Habitat of the identifier to populate witnesses - msgs (Deck): incoming messages to publish to witnesses - cues (Deck): outgoing cues of successful messages - + hby (Habery): Habitat environment for identifier lookups. + msgs (Deck): publish requests with `pre` and `msg`. + cues (Deck): completed request cues. """ self.hby = hby + self.posted = 0 self.msgs = msgs if msgs is not None else decking.Deck() self.cues = cues if cues is not None else decking.Deck() super(WitnessPublisher, self).__init__(doers=[doing.doify(self.sendDo)], **kwa) - def sendDo(self, tymth=None, tock=0.0, **opts): - """ - Returns doifiable Doist compatible generator method (doer dog) + def sendDo(self, tymth=None, tock=None, **opts): + """Doer loop that sends queued messages to each witness. - Usage: - add result of doify on this method to doers list + Pushes the original request to self.cues to signal completion """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessPublisherTock _ = (yield self.tock) while True: while self.msgs: evt = self.msgs.popleft() + self.posted += 1 pre = evt["pre"] msg = evt["msg"] @@ -614,12 +591,10 @@ def sendDo(self, tymth=None, tock=0.0, **opts): _ = (yield self.tock) - total = len(witers) - count = 0 - while count < total: - for witer in witers: - count += len(witer.sent) - _ = (yield self.tock) + while witers: + witer = witers.pop() + while not witer.idle: + _ = (yield self.tock) self.remove(witers) self.cues.push(evt) @@ -641,20 +616,23 @@ def sent(self, said): return False + @property + def idle(self): + return len(self.msgs) == 0 and self.posted == len(self.cues) -class TCPMessenger(doing.DoDoer): - """ Send events to witnesses for receipting using TCP direct connection - """ +class TCPMessenger(doing.DoDoer): + """Send outbound CESR messages to a witness via TCP and parse inbound receipts.""" def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize TCP messenger with queues and parser wiring. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): tcp endpoint URL for the witness. + msgs (Deck | None): outbound message queue. + sent (Deck | None): sent message queue. """ self.hab = hab self.wit = wit @@ -671,13 +649,8 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): super(TCPMessenger, self).__init__(doers=doers) - def receiptDo(self, tymth=None, tock=0.0): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list - """ + def receiptDo(self, tymth=None, tock=0.0, **kwa): + """Doer loop that sends queued messages over TCP.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -710,26 +683,8 @@ def receiptDo(self, tymth=None, tock=0.0): yield self.tock def msgDo(self, tymth=None, tock=0.0, **opts): - """ - Returns doifiable Doist compatible generator method (doer dog) to process - incoming message stream of .kevery - - Doist Injected Attributes: - g.tock = tock # default tock attributes - g.done = None # default done state - g.opts - - Parameters: - tymth is injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock is injected initial tock value - opts is dict of injected optional additional parameters - - - Usage: - add result of doify on this method to doers list - """ - yield from self.parser.parsator() # process messages continuously + """Doer loop that parses inbound TCP messages into the Kevery.""" + yield from self.parser.parsator(local=True) # process messages continuously @property def idle(self): @@ -737,18 +692,17 @@ def idle(self): class TCPStreamMessenger(doing.DoDoer): - """ Send events to witnesses for receipting using TCP direct connection - - """ + """Stream a CESR message to a witness via TCP and parse inbound receipts.""" def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize TCP stream messenger with queues and parser wiring. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): tcp endpoint URL for the witness. + msgs (Deck | None): outbound message queue. + sent (Deck | None): sent message queue. """ self.hab = hab self.wit = wit @@ -765,12 +719,10 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): super(TCPStreamMessenger, self).__init__(doers=doers) - def receiptDo(self, tymth=None, tock=0.0): - """ - Returns doifiable Doist compatible generator method (doer dog) + def receiptDo(self, tymth=None, tock=0.0, **kwa): + """Doer loop that sends queued messages over TCP. - Usage: - add result of doify on this method to doers list + Pushes the original request to self.sent to signal completion """ self.wind(tymth) self.tock = tock @@ -804,26 +756,8 @@ def receiptDo(self, tymth=None, tock=0.0): yield self.tock def msgDo(self, tymth=None, tock=0.0, **opts): - """ - Returns doifiable Doist compatible generator method (doer dog) to process - incoming message stream of .kevery - - Doist Injected Attributes: - g.tock = tock # default tock attributes - g.done = None # default done state - g.opts - - Parameters: - tymth is injected function wrapper closure returned by .tymen() of - Tymist instance. Calling tymth() returns associated Tymist .tyme. - tock is injected initial tock value - opts is dict of injected optional additional parameters - - - Usage: - add result of doify on this method to doers list - """ - yield from self.parser.parsator() # process messages continuously + """Doer loop that parses inbound TCP messages into the Kevery.""" + yield from self.parser.parsator(local=True) # process messages continuously @property def idle(self): @@ -831,19 +765,18 @@ def idle(self): class HTTPMessenger(doing.DoDoer): - """ - Interacts with Recipients on HTTP and SSE for sending events and receiving receipts + """Send CESR messages to a witness over HTTP and capture responses.""" - """ - - def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, auth=None, **kwa): + """Initialize HTTP messenger with queues and optional auth. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): http/https endpoint URL for the witness. + msgs (Deck | None): outbound message queue. + sent (Deck | None): response queue. + auth (str | None): optional 2FA auth codes for witnesses. """ self.hab = hab self.wit = wit @@ -851,6 +784,7 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): self.msgs = msgs if msgs is not None else decking.Deck() self.sent = sent if sent is not None else decking.Deck() self.parser = None + self.auth = auth doers = doers if doers is not None else [] doers.extend([doing.doify(self.msgDo), doing.doify(self.responseDo)]) @@ -865,13 +799,8 @@ def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa): super(HTTPMessenger, self).__init__(doers=doers, **kwa) - def msgDo(self, tymth=None, tock=0.0): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list - """ + def msgDo(self, tymth=None, tock=0.0, **kwa): + """Doer loop that sends queued messages over HTTP.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -881,17 +810,18 @@ def msgDo(self, tymth=None, tock=0.0): yield self.tock msg = self.msgs.popleft() - self.posted += httping.streamCESRRequests(client=self.client, dest=self.wit, ims=msg) + headers = dict() + if self.auth is not None: + headers["Authorization"] = self.auth + + self.posted += httping.streamCESRRequests(client=self.client, dest=self.wit, ims=msg, headers=headers) while self.client.requests: yield self.tock yield self.tock - def responseDo(self, tymth=None, tock=0.0): - """ - Processes responses from client and adds them to sent cue - - """ + def responseDo(self, tymth=None, tock=0.0, **kwa): + """Doer loop that processes HTTP responses from the client and adds them into `sent` cues.""" self.wind(tymth) self.tock = tock _ = (yield self.tock) @@ -909,19 +839,17 @@ def idle(self): class HTTPStreamMessenger(doing.DoDoer): - """ - Interacts with Recipients on HTTP and SSE for sending events and receiving receipts - - """ + """Send a single CESR message via HTTP PUT and capture the response.""" def __init__(self, hab, wit, url, msg=b'', headers=None, **kwa): - """ - For the current event, gather the current set of witnesses, send the event, - gather all receipts and send them to all other witnesses + """Initialize a single-request HTTP messenger. Parameters: - hab: Habitat of the identifier to populate witnesses - + hab (Hab): habitat for KEL parsing and db access. + wit (str): qb64 witness identifier. + url (str): http/https endpoint URL for the witness. + msg (bytes): CESR message body to send. + headers (dict | None): extra HTTP headers. """ self.hab = hab self.wit = wit @@ -953,12 +881,7 @@ def __init__(self, hab, wit, url, msg=b'', headers=None, **kwa): super(HTTPStreamMessenger, self).__init__(doers=doers, **kwa) def recur(self, tyme, deeds=None): - """ - Returns doifiable Doist compatible generator method (doer dog) - - Usage: - add result of doify on this method to doers list - """ + """Poll for a response and stop once received.""" if self.client.responses: self.rep = self.client.respond() self.remove([self.client]) @@ -983,34 +906,36 @@ def mailbox(hab, cid): return mbx -def messenger(hab, pre): +def messenger(hab, pre, auth=None): """ Create a Messenger (tcp or http) based on available endpoints Parameters: hab (Habitat): Environment to use to look up witness URLs pre (str): qb64 identifier prefix of recipient to create a messanger for + auth (str): optional auth code to send with any request for messenger Returns: Optional(TcpWitnesser, HTTPMessenger): witnesser for ensuring full reciepts """ urls = hab.fetchUrls(eid=pre) - return messengerFrom(hab, pre, urls) + return messengerFrom(hab, pre, urls, auth) -def messengerFrom(hab, pre, urls): +def messengerFrom(hab, pre, urls, auth=None): """ Create a Witnesser (tcp or http) based on provided endpoints Parameters: hab (Habitat): Environment to use to look up witness URLs pre (str): qb64 identifier prefix of recipient to create a messanger for urls (dict): map of schemes to urls of available endpoints + auth (str): optional auth code to send with any request for messenger Returns: Optional(TcpWitnesser, HTTPMessenger): witnesser for ensuring full reciepts """ if kering.Schemes.http in urls or kering.Schemes.https in urls: url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - witer = HTTPMessenger(hab=hab, wit=pre, url=url) + witer = HTTPMessenger(hab=hab, wit=pre, url=url, auth=auth) elif kering.Schemes.tcp in urls: url = urls[kering.Schemes.tcp] witer = TCPMessenger(hab=hab, wit=pre, url=url) @@ -1021,7 +946,7 @@ def messengerFrom(hab, pre, urls): def streamMessengerFrom(hab, pre, urls, msg, headers=None): - """ Create a Witnesser (tcp or http) based on provided endpoints + """Create a stream messenger (HTTP or TCP) for a single outbound message. Parameters: hab (Habitat): Environment to use to look up witness URLs @@ -1063,7 +988,7 @@ def httpClient(hab, wit): url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(url) - client = http.clienting.Client(scheme=up.scheme, hostname=up.hostname, port=up.port) + client = http.clienting.Client(scheme=up.scheme, hostname=up.hostname, port=up.port, path=up.path) clientDoer = http.clienting.ClientDoer(client=client) return client, clientDoer diff --git a/src/keri/app/apping.py b/src/keri/app/apping.py index 4464633db..95d77249b 100644 --- a/src/keri/app/apping.py +++ b/src/keri/app/apping.py @@ -4,6 +4,8 @@ keri.app.apping module """ +import cmd + from hio.base import doing from hio.core.serial import serialing @@ -26,7 +28,7 @@ def __init__(self, db=None, console=None, **kwa): self.db = db if db is not None else basing.Baser() self.console = console if console is not None else serialing.Console() - def enter(self): + def enter(self, *, temp=None): """""" if not self.console.reopen(): raise IOError("Unable to open serial console.") diff --git a/src/keri/app/cli/commands/aid.py b/src/keri/app/cli/commands/aid.py new file mode 100644 index 000000000..953e463e8 --- /dev/null +++ b/src/keri/app/cli/commands/aid.py @@ -0,0 +1,58 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Print the AID for a given alias') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None, + required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(status, **kwa)] + + +def status(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + if hab is None: + print(f"{alias} is not a valid alias for an identifier") + + print(hab.pre) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/challenge/respond.py b/src/keri/app/cli/commands/challenge/respond.py index 75c31094a..d942312da 100644 --- a/src/keri/app/cli/commands/challenge/respond.py +++ b/src/keri/app/cli/commands/challenge/respond.py @@ -7,6 +7,7 @@ from hio.base import doing +from keri import help from keri.app import habbing, forwarding, connecting from keri.app.cli.common import existing from keri.app.habbing import GroupHab @@ -19,13 +20,14 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--words', '-d', help='JSON formatted array of words to sign, \'@\' allowed to load from a file', action="store", required=True) parser.add_argument('--recipient', '-r', help='Contact alias of the AID to send the signed words to', action="store", required=True) +logger = help.ogler.getLogger() def respond(args): """ diff --git a/src/keri/app/cli/commands/challenge/verify.py b/src/keri/app/cli/commands/challenge/verify.py index f82fbf1c4..eed9d285b 100644 --- a/src/keri/app/cli/commands/challenge/verify.py +++ b/src/keri/app/cli/commands/challenge/verify.py @@ -28,7 +28,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--words', '-d', help='JSON formatted array of words to verfiy, \'@\' allowed to load from a file', @@ -88,7 +88,7 @@ def __init__(self, name, alias, base, bran, words, generate, strength, out, sign super(VerifyDoer, self).__init__(doers=doers) - def verifyDo(self, tymth, tock=0.0): + def verifyDo(self, tymth, tock=0.0, **kwa): """ Check for any credential messages in mailboxes and list all held credentials Parameters: diff --git a/src/keri/app/cli/commands/clean.py b/src/keri/app/cli/commands/clean.py new file mode 100644 index 000000000..95bdd6e81 --- /dev/null +++ b/src/keri/app/cli/commands/clean.py @@ -0,0 +1,64 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app.cli.common import existing + +logger = help.ogler.getLogger() + + +def handler(args): + """ + Launch KERI database initialization + + Args: + args(Namespace): arguments object from command line + """ + clean = CleanDoer(args) + return [clean] + + +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore') +parser.set_defaults(handler=handler, + transferable=True) + +# Parameters for basic structure of database +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--temp', '-t', help='create a temporary keystore, used for testing', default=False) + +# Parameters for Manager creation +# passcode => bran +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +class CleanDoer(doing.Doer): + + def __init__(self, args): + self.args = args + super(CleanDoer, self).__init__() + + def recur(self, tyme): + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + print("Migrating...") + hby.db.migrate() + print("Finished") + + hby = existing.setupHby(name=self.args.name, base=self.args.base, + bran=self.args.bran, temp=self.args.temp) + + print("Database open, performing clean...") + hby.db.clean() + print("Finished.") + + return True diff --git a/src/keri/app/cli/commands/contacts/add.py b/src/keri/app/cli/commands/contacts/add.py new file mode 100644 index 000000000..aca6fb08d --- /dev/null +++ b/src/keri/app/cli/commands/contacts/add.py @@ -0,0 +1,118 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.add module + +""" +import argparse +import json + +from keri import help +from hio.base import doing + +import keri.app.oobiing +from keri.app import connecting, habbing, oobiing +from keri.app.cli.common import existing +from keri.db import basing +from keri.help import helping + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add a contact via OOBI resolution') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--oobi', '-o', help='OOBI URL to resolve for contact', required=True) +parser.add_argument('--alias', help='alias to set for contact', required=False, default=None) +parser.add_argument('--field', '-f', help='field in key=value format', action='append', dest='fields', default=None) + + +def handler(args): + """ command line method for adding a contact via OOBI resolution + + Parameters: + args(Namespace): parse args namespace object + + """ + name = args.name + base = args.base + bran = args.bran + oobi = args.oobi + alias = args.alias + fields = args.fields + + addDoer = ContactAddDoer(name=name, base=base, bran=bran, + oobi=oobi, alias=alias, fields=fields) + return [addDoer] + + +class ContactAddDoer(doing.DoDoer): + """ DoDoer for adding a contact via OOBI resolution """ + + def __init__(self, name, base, bran, oobi, alias, fields): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) + self.oobi = oobi + self.alias = alias + self.fields = fields + + doers = [self.hbyDoer, doing.doify(self.add)] + super(ContactAddDoer, self).__init__(doers=doers) + + def add(self, tymth, tock=0.0, **kwa): + """ Resolves OOBI and creates contact + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + obr = basing.OobiRecord(date=helping.nowIso8601()) + if self.alias: + obr.oobialias = self.alias + + self.hby.db.oobis.put(keys=(self.oobi,), val=obr) + + obi = keri.app.oobiing.Oobiery(hby=self.hby) + authn = oobiing.Authenticator(hby=self.hby) + self.extend(obi.doers) + self.extend(authn.doers) + + # Wait for resolution + while not self.hby.db.roobi.get(keys=(self.oobi,)): + yield 0.25 + + resolved = self.hby.db.roobi.get(keys=(self.oobi,)) + cid = resolved.cid + + org = connecting.Organizer(hby=self.hby) + + data = {} + if self.alias: + data['alias'] = self.alias + + if self.fields: + for field in self.fields: + if '=' not in field: + print(f"Invalid field format: {field}. Use key=value") + self.remove([self.hbyDoer, *obi.doers, *authn.doers]) + return + key, val = field.split('=', 1) + data[key] = val + + org.update(cid, data) + + contact = org.get(cid) + print(json.dumps(contact, indent=2)) + + self.remove([self.hbyDoer, *obi.doers, *authn.doers]) diff --git a/src/keri/app/cli/commands/contacts/delete.py b/src/keri/app/cli/commands/contacts/delete.py new file mode 100644 index 000000000..1dfc55a05 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/delete.py @@ -0,0 +1,91 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.delete module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Delete a contact') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--aid', '-a', help='contact AID prefix', required=False, default=None) +parser.add_argument('--alias', help='contact alias to lookup', required=False, default=None) +parser.add_argument('--yes', '-y', help='skip confirmation', action='store_true', default=False) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(delete, **kwa)] + + +def delete(tymth, tock=0.0, **opts): + """ Command line handler for deleting a contact + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + aid = args.aid + alias = args.alias + yes = args.yes + + if aid is None and alias is None: + print("Either --aid or --alias is required") + return -1 + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contact = None + pre = None + + if aid: + pre = aid + contact = org.get(aid) + elif alias: + contacts = org.find('alias', f"^{alias}$") # Exact match + if len(contacts) == 0: + print(f"Contact with alias '{alias}' not found") + return -1 + if len(contacts) > 1: + print(f"Multiple contacts match alias '{alias}'") + return -1 + contact = contacts[0] + pre = contact['id'] + + if contact is None: + print("Contact not found") + return -1 + + alias_display = contact.get('alias', pre) + + if not yes: + confirm = input(f"Delete contact '{alias_display}' ({pre})? [y/N]: ") + if confirm.lower() != 'y': + print("Aborted") + return 0 + + org.rem(pre) + print(f"Deleted contact '{alias_display}' ({pre})") + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/src/keri/app/cli/commands/contacts/find.py b/src/keri/app/cli/commands/contacts/find.py new file mode 100644 index 000000000..41befd719 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/find.py @@ -0,0 +1,62 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.find module + +""" +import argparse +import json + +from keri import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Find contacts by field value') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--field', '-f', help='field name to search (default: alias)', default='alias') +parser.add_argument('--value', '-v', help='value or regex pattern to match', required=True) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(find, **kwa)] + + +def find(tymth, tock=0.0, **opts): + """ Command line handler for finding contacts by field value + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + field = args.field + value = args.value + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contacts = org.find(field, value) + + if len(contacts) == 0: + print(f"No contacts found matching {field}='{value}'") + return + + print(json.dumps(contacts, indent=2)) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/src/keri/app/cli/commands/contacts/get.py b/src/keri/app/cli/commands/contacts/get.py new file mode 100644 index 000000000..6d3cbc9a4 --- /dev/null +++ b/src/keri/app/cli/commands/contacts/get.py @@ -0,0 +1,104 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.get module + +""" +import argparse +import json + +from keri import help +from hio.base import doing +from keri import kering + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Get a single contact') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--aid', '-a', help='contact AID prefix', required=False, default=None) +parser.add_argument('--alias', help='contact alias to lookup', required=False, default=None) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(get, **kwa)] + + +def get(tymth, tock=0.0, **opts): + """ Command line handler for getting a single contact + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + aid = args.aid + alias = args.alias + + if aid is None and alias is None: + print("Either --aid or --alias is required") + return -1 + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contact = None + pre = None + + if aid: + pre = aid + contact = org.get(aid) + elif alias: + contacts = org.find('alias', f"^{alias}$") # Exact match + if len(contacts) > 1: + print(f"Multiple contacts match alias '{alias}'") + return -1 + if len(contacts) == 1: + contact = contacts[0] + pre = contact['id'] + + if contact is None: + print("Contact not found") + return -1 + + accepted = [saider.qb64 for saider in hby.db.chas.get(keys=(pre,))] + received = [saider.qb64 for saider in hby.db.reps.get(keys=(pre,))] + valid = set(accepted) & set(received) + + challenges = [] + for said in valid: + try: + exn = hby.db.exns.get(keys=(said,)) + except kering.ValidationError: + val = hby.db.getVal(db=hby.db.exns.sdb, key=hby.db.exns._tokey((said,))) + d = json.loads(bytes(val).decode("utf-8")) + challenges.append(dict(dt=d['dt'], words=d['a']['words'])) + else: + challenges.append(dict(dt=exn.ked['dt'], words=exn.ked['a']['words'])) + + contact["challenges"] = challenges + + wellKnowns = [] + wkans = hby.db.wkas.get(keys=(pre,)) + for wkan in wkans: + wellKnowns.append(dict(url=wkan.url, dt=wkan.dt)) + + contact["wellKnowns"] = wellKnowns + + print(json.dumps(contact, indent=2)) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/src/keri/app/cli/commands/contacts/list.py b/src/keri/app/cli/commands/contacts/list.py index b93c1e248..5daf4d411 100644 --- a/src/keri/app/cli/commands/contacts/list.py +++ b/src/keri/app/cli/commands/contacts/list.py @@ -23,7 +23,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/contacts/query.py b/src/keri/app/cli/commands/contacts/query.py new file mode 100644 index 000000000..c5ae345ed --- /dev/null +++ b/src/keri/app/cli/commands/contacts/query.py @@ -0,0 +1,135 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.query module + +""" +import argparse +import datetime +import json + +from keri import help +from hio.base import doing +from hio.help import decking + +from keri.app import connecting, habbing, indirecting, querying +from keri.app.cli.common import existing, displaying +from keri.help import helping + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Query witnesses for latest key state of a contact') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--alias', '-a', help='alias of your local identifier for querying', required=True) +parser.add_argument('--contact-aid', dest='contact_aid', help='contact AID prefix to query', + required=False, default=None) +parser.add_argument('--contact-alias', dest='contact_alias', help='contact alias to lookup', + required=False, default=None) + + +def handler(args): + """ command line method for querying contact key state from witnesses + + Parameters: + args(Namespace): parse args namespace object + + """ + name = args.name + base = args.base + bran = args.bran + alias = args.alias + contact_aid = args.contact_aid + contact_alias = args.contact_alias + + if contact_aid is None and contact_alias is None: + print("Either --contact-aid or --contact-alias is required") + return [] + + queryDoer = ContactQueryDoer(name=name, base=base, bran=bran, alias=alias, + contact_aid=contact_aid, contact_alias=contact_alias) + return [queryDoer] + + +class ContactQueryDoer(doing.DoDoer): + """ DoDoer for querying contact key state from witnesses """ + + def __init__(self, name, base, bran, alias, contact_aid, contact_alias): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) + self.alias = alias + self.contact_aid = contact_aid + self.contact_alias = contact_alias + + self.mbd = indirecting.MailboxDirector(hby=self.hby, topics=["/replay", "/receipt", "/reply"]) + + doers = [self.hbyDoer, self.mbd, doing.doify(self.queryDo)] + super(ContactQueryDoer, self).__init__(doers=doers) + + def queryDo(self, tymth, tock=0.0, **kwa): + """ Query witnesses for contact key state + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + hab = self.hby.habByName(self.alias) + if hab is None: + print(f"Local identifier '{self.alias}' not found") + self.remove([self.hbyDoer, self.mbd]) + return + + org = connecting.Organizer(hby=self.hby) + + pre = None + if self.contact_aid: + pre = self.contact_aid + contact = org.get(pre) + elif self.contact_alias: + contacts = org.find('alias', f"^{self.contact_alias}$") + if len(contacts) == 0: + print(f"Contact with alias '{self.contact_alias}' not found") + self.remove([self.hbyDoer, self.mbd]) + return + if len(contacts) > 1: + print(f"Multiple contacts match alias '{self.contact_alias}'") + self.remove([self.hbyDoer, self.mbd]) + return + contact = contacts[0] + pre = contact['id'] + + if pre is None: + print("Contact not found") + self.remove([self.hbyDoer, self.mbd]) + return + + doer = querying.QueryDoer(hby=self.hby, hab=hab, pre=pre, kvy=self.mbd.kvy) + self.extend([doer]) + + end = helping.nowUTC() + datetime.timedelta(seconds=10) + while helping.nowUTC() < end: + if doer.done: + break + yield 1.0 + + self.remove([doer]) + + displaying.printExternal(self.hby, pre) + + contact = org.get(pre) + if contact: + print(json.dumps(contact, indent=2)) + + self.remove([self.hbyDoer, self.mbd]) diff --git a/src/keri/app/cli/commands/contacts/rename.py b/src/keri/app/cli/commands/contacts/rename.py new file mode 100644 index 000000000..b8c164add --- /dev/null +++ b/src/keri/app/cli/commands/contacts/rename.py @@ -0,0 +1,84 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands.contacts.rename module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app import connecting +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Rename a contact alias') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--aid', '-a', help='contact AID prefix', required=False, default=None) +parser.add_argument('--old-alias', dest='old_alias', help='current alias to lookup', required=False, default=None) +parser.add_argument('--alias', help='new alias to set', required=True) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(rename, **kwa)] + + +def rename(tymth, tock=0.0, **opts): + """ Command line handler for renaming a contact alias + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + aid = args.aid + old_alias = args.old_alias + new_alias = args.alias + + if aid is None and old_alias is None: + print("Either --aid or --old-alias is required") + return -1 + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + + contact = None + pre = None + + if aid: + pre = aid + contact = org.get(aid) + elif old_alias: + contacts = org.find('alias', f"^{old_alias}$") # Exact match + if len(contacts) == 0: + print(f"Contact with alias '{old_alias}' not found") + return -1 + if len(contacts) > 1: + print(f"Multiple contacts match alias '{old_alias}'") + return -1 + contact = contacts[0] + pre = contact['id'] + + if contact is None: + print("Contact not found") + return -1 + + old_alias_display = contact.get('alias', '') + org.set(pre, 'alias', new_alias) + print(f"Renamed contact {pre} from '{old_alias_display}' to '{new_alias}'") + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first") + return -1 diff --git a/src/keri/app/cli/commands/decrypt.py b/src/keri/app/cli/commands/decrypt.py new file mode 100644 index 000000000..063242c4d --- /dev/null +++ b/src/keri/app/cli/commands/decrypt.py @@ -0,0 +1,65 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio.base import doing + +from keri import kering +from keri.app.cli.common import existing +from keri.core import indexing, coring, MtrDex + +parser = argparse.ArgumentParser(description='Decrypt arbitrary data for AIDs with Ed25519 public keys only') +parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--data', '-d', help='Encrypted data or file (starts with "@")', required=True) + + +def handler(args): + """ + Verify signatures on arbitrary data + + Args: + args(Namespace): arguments object from command line + """ + kwa = dict(args=args) + return [doing.doify(decrypt, **kwa)] + + +def decrypt(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (_, hab): + + data = args.data + if data.startswith("@"): + f = open(data[1:], "r") + data = f.read() + else: + data = data + + m = coring.Matter(qb64=data) # should refactor this to use Cipher + d = coring.Matter(qb64=hab.decrypt(ser=m.raw)) + print(d.raw) + + except kering.ConfigurationError: + print(f"prefix for {name} does not exist, incept must be run first", ) + except FileNotFoundError: + print("unable to open file", args.text[1:]) diff --git a/src/keri/app/cli/commands/delegate/confirm.py b/src/keri/app/cli/commands/delegate/confirm.py index 37333150c..034ae24a2 100644 --- a/src/keri/app/cli/commands/delegate/confirm.py +++ b/src/keri/app/cli/commands/delegate/confirm.py @@ -13,8 +13,10 @@ from keri.app import habbing, indirecting, agenting, grouping, forwarding, delegating, notifying from keri.app.cli.common import existing from keri.app.habbing import GroupHab +from keri import core from keri.core import coring, serdering from keri.db import dbing +from keri.help import helping from keri.peer import exchanging logger = help.ogler.getLogger() @@ -26,11 +28,16 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--interact", "-i", help="anchor the delegation approval in an interaction event. " "Default is to use a rotation event.", action="store_true") parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') +parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', + default=[], action="append", required=False) +parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False) def confirm(args): @@ -46,15 +53,20 @@ def confirm(args): alias = args.alias interact = args.interact auto = args.auto + authenticate = args.authenticate + codes = args.code + codeTime = args.code_time - confirmDoer = ConfirmDoer(name=name, base=base, alias=alias, bran=bran, interact=interact, auto=auto) + confirmDoer = ConfirmDoer(name=name, base=base, alias=alias, bran=bran, interact=interact, auto=auto, + authenticate=authenticate, codes=codes, codeTime=codeTime) doers = [confirmDoer] return doers class ConfirmDoer(doing.DoDoer): - def __init__(self, name, base, alias, bran, interact=False, auto=False): + def __init__(self, name, base, alias, bran, interact=False, auto=False, authenticate=False, codes=None, + codeTime=None): hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer self.witq = agenting.WitnessInquisitor(hby=hby) @@ -62,6 +74,9 @@ def __init__(self, name, base, alias, bran, interact=False, auto=False): self.counselor = grouping.Counselor(hby=hby) self.notifier = notifying.Notifier(hby=hby) self.mux = grouping.Multiplexor(hby=hby, notifier=self.notifier) + self.authenticate = authenticate + self.codes = codes if codes is not None else [] + self.codeTime = codeTime exc = exchanging.Exchanger(hby=hby, handlers=[]) delegating.loadHandlers(hby=hby, exc=exc, notifier=self.notifier) @@ -79,7 +94,15 @@ def __init__(self, name, base, alias, bran, interact=False, auto=False): self.auto = auto super(ConfirmDoer, self).__init__(doers=doers) - def confirmDo(self, tymth, tock=0.0): + def _addAuthorizerSeal(self, pre, edig, anchorSn, anchorSaid): + """Save the authorizer (delegator) event seal of the anchoring IXN event for an approved delegation.""" + dgkey = dbing.dgKey(pre, edig) + seqner = coring.Seqner(sn=anchorSn) + saider = coring.Saider(qb64=anchorSaid) + couple = seqner.qb64b + saider.qb64b + self.hby.db.setAes(dgkey, couple) + + def confirmDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -95,9 +118,8 @@ def confirmDo(self, tymth, tock=0.0): while True: esc = self.escrowed() - for ekey, edig in esc: - pre, sn = dbing.splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dbing.dgKey(pre, bytes(edig)) + for pre, sn, edig in esc: + dgkey = dbing.dgKey(pre, edig) eraw = self.hby.db.getEvt(dgkey) if eraw is None: continue @@ -111,7 +133,7 @@ def confirmDo(self, tymth, tock=0.0): elif ilk in (coring.Ilks.drt,): typ = "rotation" dkever = self.hby.kevers[eserder.pre] - delpre = dkever.delegator + delpre = dkever.delpre else: continue @@ -130,8 +152,8 @@ def confirmDo(self, tymth, tock=0.0): if isinstance(hab, GroupHab): aids = hab.smids - seqner = coring.Seqner(sn=eserder.sn) - anchor = dict(i=eserder.ked["i"], s=seqner.snh, d=eserder.said) + + anchor = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) if self.interact: msg = hab.interact(data=[anchor]) else: @@ -148,12 +170,12 @@ def confirmDo(self, tymth, tock=0.0): attachment=atc) prefixer = coring.Prefixer(qb64=hab.pre) - seqner = coring.Seqner(sn=serder.sn) + sner = core.Number(num=serder.sn, code=core.NumDex.Huge) # maybe serder.sner instead so not Huge saider = coring.Saider(qb64b=serder.saidb) - self.counselor.start(ghab=hab, prefixer=prefixer, seqner=seqner, saider=saider) + self.counselor.start(ghab=hab, prefixer=prefixer, seqner=sner, saider=saider) while True: - saider = self.hby.db.cgms.get(keys=(prefixer.qb64, seqner.qb64)) + saider = self.hby.db.cgms.get(keys=(prefixer.qb64, sner.qb64)) if saider is not None: break @@ -161,19 +183,36 @@ def confirmDo(self, tymth, tock=0.0): print(f"Delegate {eserder.pre} {typ} event committed.") + self._addAuthorizerSeal(pre, edig, anchorSn=serder.sn, + anchorSaid=serder.said) + self.hby.kvy.processEscrowDelegables() # removes DIP/DRT from delegables after adding it to kevers self.remove(self.toRemove) return True else: cur = hab.kever.sner.num - seqner = coring.Seqner(sn=eserder.sn) - anchor = dict(i=eserder.ked["i"], s=seqner.snh, d=eserder.said) + + anchor = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) if self.interact: hab.interact(data=[anchor]) else: hab.rotate(data=[anchor]) - witDoer = agenting.WitnessReceiptor(hby=self.hby) + auths = {} + if self.authenticate: + codeTime = helping.fromIso8601( + self.codeTime) if self.codeTime is not None else helping.nowIso8601() + for arg in self.codes: + (wit, code) = arg.split(":") + auths[wit] = f"{code}#{codeTime}" + + for wit in hab.kever.wits: + if wit in auths: + continue + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + + witDoer = agenting.WitnessReceiptor(hby=self.hby, auths=auths) self.extend(doers=[witDoer]) self.toRemove.append(witDoer) yield self.tock @@ -203,6 +242,9 @@ def confirmDo(self, tymth, tock=0.0): print(f"Delegate {eserder.pre} {typ} event committed.") + self._addAuthorizerSeal(pre, edig, anchorSn=hab.kever.sn, + anchorSaid=hab.kever.serder.said) + self.hby.kvy.processEscrowDelegables() # removes DIP/DRT from delegables after adding it to kevers self.remove(self.toRemove) return True @@ -212,12 +254,6 @@ def confirmDo(self, tymth, tock=0.0): def escrowed(self): esc = [] - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.hby.db.getPseItemsNextIter(key=key): - esc.append((ekey, edig)) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey - + for (pre, sn), edig in self.hby.db.delegables.getItemIter(): + esc.append((pre, sn, edig)) return esc diff --git a/src/keri/app/cli/commands/delegate/request.py b/src/keri/app/cli/commands/delegate/request.py index 85d6cbd0f..560d6014f 100644 --- a/src/keri/app/cli/commands/delegate/request.py +++ b/src/keri/app/cli/commands/delegate/request.py @@ -25,7 +25,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran def request(args): @@ -62,7 +62,7 @@ def __init__(self, name, base, alias, bran): super(RequestDoer, self).__init__(doers=doers) - def requestDo(self, tymth, tock=0.0): + def requestDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -86,7 +86,7 @@ def requestDo(self, tymth, tock=0.0): (seqner, saider) = esc[0] evt = hab.makeOwnEvent(sn=seqner.sn) - delpre = hab.kever.delegator # get the delegator identifier + delpre = hab.kever.delpre # get the delegator identifier if isinstance(hab, GroupHab): phab = hab.mhab @@ -96,10 +96,10 @@ def requestDo(self, tymth, tock=0.0): exn, atc = delegating.delegateRequestExn(hab.mhab, delpre=delpre, evt=bytes(evt), aids=hab.smids) # delegate AID ICP and exn of delegation request EXN - srdr = serdering.SerderKERI(raw=evt) # coring.Serder(raw=evt) + srdr = serdering.SerderKERI(raw=evt) del evt[:srdr.size] self.postman.send(src=phab.pre, dest=delpre, topic="delegate", serder=srdr, attachment=evt) - self.postman.send(src=phab.pre, dest=hab.kever.delegator, topic="delegate", serder=exn, attachment=atc) + self.postman.send(src=phab.pre, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) while True: while self.postman.cues: diff --git a/src/keri/app/cli/commands/did/generate.py b/src/keri/app/cli/commands/did/generate.py index 5398ca35b..d373e6a61 100644 --- a/src/keri/app/cli/commands/did/generate.py +++ b/src/keri/app/cli/commands/did/generate.py @@ -30,7 +30,7 @@ # Parameters for Manager access # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) parser.add_argument('--url', '-u', help="generate a DID URL instead of a DID", action="store_true") @@ -73,20 +73,20 @@ def generate(tymth, tock=0.0, **opts): sys.exit(-1) wit = random.choice(hab.kever.wits) - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise kering.ConfigurationError(f"unable to query witness {wit}, no http endpoint") - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - enc = urllib.parse.quote_plus(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + enc = urllib.parse.quote_plus(f"{url.rstrip("/")}/oobi/{hab.pre}/witness") print(f"did:keri:{hab.pre}?oobi={enc}") elif role in (kering.Roles.controller,): - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: print(f"{alias} identifier {hab.pre} does not have any controller endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - enc = urllib.parse.quote_plus(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + enc = urllib.parse.quote_plus(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") print(f"did:keri:{hab.pre}?oobi={enc}") diff --git a/src/keri/app/cli/commands/ends/add.py b/src/keri/app/cli/commands/ends/add.py index 76cb623ec..90a1aa11c 100644 --- a/src/keri/app/cli/commands/ends/add.py +++ b/src/keri/app/cli/commands/ends/add.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--role", "-r", help="KERI enpoint authorization role.", required=True) @@ -75,7 +75,7 @@ def __init__(self, name, base, alias, bran, role, eid, timestamp=None): super(RoleDoer, self).__init__(doers=self.toRemove + [doing.doify(self.roleDo)]) - def roleDo(self, tymth, tock=0.0): + def roleDo(self, tymth, tock=0.0, **kwa): """ Export any end reply messages previous saved for the provided AID Parameters: diff --git a/src/keri/app/cli/commands/ends/export.py b/src/keri/app/cli/commands/ends/export.py index e340272d8..415842378 100644 --- a/src/keri/app/cli/commands/ends/export.py +++ b/src/keri/app/cli/commands/ends/export.py @@ -11,7 +11,7 @@ from keri import kering from keri.app.cli.common import existing -from keri.core import eventing +from keri.core import coring, eventing logger = help.ogler.getLogger() @@ -21,7 +21,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--aid", "-a", help="qualified base64 of AID to export rpy messages for all endpoints.", @@ -50,7 +50,7 @@ def __init__(self, name, base, bran, aid): super(ExportDoer, self).__init__(doers=doers) - def exportDo(self, tymth, tock=0.0): + def exportDo(self, tymth, tock=0.0, **kwa): """ Export any end reply messages previous saved for the provided AID Parameters: diff --git a/src/keri/app/cli/commands/ends/list.py b/src/keri/app/cli/commands/ends/list.py index 1f0035a3b..0cdf8da29 100644 --- a/src/keri/app/cli/commands/ends/list.py +++ b/src/keri/app/cli/commands/ends/list.py @@ -11,7 +11,9 @@ from hio.base import doing from keri import kering +from keri.app import indirecting, habbing, forwarding, grouping from keri.app.cli.common import existing +from keri.core import eventing, parsing, coring logger = help.ogler.getLogger() @@ -22,7 +24,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--aid", help="qualified base64 of AID to export rpy messages for all endpoints.", required=True) @@ -53,7 +55,7 @@ def __init__(self, name, base, alias, bran, aid): super(RoleDoer, self).__init__(doers=doers) - def roleDo(self, tymth, tock=0.0): + def roleDo(self, tymth, tock=0.0, **kwa): """ Export any end reply messages previous saved for the provided AID Parameters: diff --git a/src/keri/app/cli/commands/escrow.py b/src/keri/app/cli/commands/escrow.py new file mode 100644 index 000000000..43d3dda62 --- /dev/null +++ b/src/keri/app/cli/commands/escrow.py @@ -0,0 +1,159 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import json + +from keri import help +from hio.base import doing + +from keri.core import eventing +from keri.app.cli.common import existing +from keri.db import dbing +from keri.kering import ConfigurationError +from keri.vdr import viring + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Views events in escrow state.') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + +parser.add_argument("--escrow", "-e", help="show values for one specific escrow", default=None) + + +def handler(args): + """ Command line escrow handler + + """ + kwa = dict(args=args) + return [doing.doify(escrows, **kwa)] + + +def escrows(tymth, tock=0.0, **opts): + _ = (yield tock) + + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + escrow = args.escrow + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + reger = viring.Reger(name=hby.name, db=hby.db, temp=False) + + escrows = dict() + if (not escrow) or escrow == "out-of-order-events": + oots = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: + for ekey, edig in hby.db.getOoeItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + oots.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["out-of-order-events"] = oots + + if (not escrow) or escrow == "partially-witnessed-events": + pwes = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in hby.db.getPweItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + pwes.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["partially-witnessed-events"] = pwes + + if (not escrow) or escrow == "partially-signed-events": + pses = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in hby.db.getPseItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + pses.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["partially-signed-events"] = pses + + if (not escrow) or escrow == "likely-duplicitous-events": + ldes = list() + key = ekey = b'' # both start same. when not same means escrows found + while True: # break when done + for ekey, edig in hby.db.getLdeItemIter(key=key): + pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item + + try: + ldes.append(eventing.loadEvent(hby.db, pre, edig)) + except ValueError as e: + raise e + + if ekey == key: # still same so no escrows found on last while iteration + break + key = ekey # setup next while iteration, with key after ekey + + escrows["likely-duplicitous-events"] = ldes + + if (not escrow) or escrow == "missing-registry-escrow": + creds = list() + for (said,), dater in reger.mre.getItemIter(): + creder, *_ = reger.cloneCred(said) + creds.append(creder.sad) + + escrows["missing-registry-escrow"] = creds + + if (not escrow) or escrow == "broken-chain-escrow": + creds = list() + for (said,), dater in reger.mce.getItemIter(): + creder, *_ = reger.cloneCred(said) + creds.append(creder.sad) + + escrows["broken-chain-escrow"] = creds + + if (not escrow) or escrow == "missing-schema-escrow": + creds = list() + for (said,), dater in reger.mse.getItemIter(): + creder, *_ = reger.cloneCred(said) + creds.append(creder.sad) + + escrows["missing-schema-escrow"] = creds + + print(json.dumps(escrows, indent=2)) + + if not(escrow) or escrow == 'tel-partial-witness-escrow': + for (regk, snq), (prefixer, seqner, saider) in reger.tpwe.getItemIter(): + pass + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/escrow/__init__.py b/src/keri/app/cli/commands/escrow/__init__.py index e69de29bb..defc725e9 100644 --- a/src/keri/app/cli/commands/escrow/__init__.py +++ b/src/keri/app/cli/commands/escrow/__init__.py @@ -0,0 +1,3 @@ +import argparse + +parser = argparse.ArgumentParser(description="A collection of escrow operations") \ No newline at end of file diff --git a/src/keri/app/cli/commands/escrow/clear.py b/src/keri/app/cli/commands/escrow/clear.py index e6e98924e..dbf08e986 100644 --- a/src/keri/app/cli/commands/escrow/clear.py +++ b/src/keri/app/cli/commands/escrow/clear.py @@ -6,8 +6,8 @@ """ import argparse -from keri import help from hio.base import doing +from keri import help from keri.app.cli.common import existing from keri.vdr import viring @@ -21,7 +21,7 @@ required=False, default="") parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran -parser.add_argument('--force', '-f', action="store_true", required=False, +parser.add_argument('--force', action="store_true", required=False, help='True means perform clear without prompting the user') @@ -54,4 +54,3 @@ def clear(tymth, tock=0.0, **opts): hby.db.clearEscrows() reger = viring.Reger(name=hby.name, db=hby.db, temp=False) reger.clearEscrows() - diff --git a/src/keri/app/cli/commands/escrow/list.py b/src/keri/app/cli/commands/escrow/list.py index dd4aa8c5c..829d59c02 100644 --- a/src/keri/app/cli/commands/escrow/list.py +++ b/src/keri/app/cli/commands/escrow/list.py @@ -7,9 +7,9 @@ import argparse import json -from keri import help from hio.base import doing +from keri import help from keri.core import eventing from keri.app.cli.common import existing from keri.db import dbing @@ -49,9 +49,11 @@ def escrows(tymth, tock=0.0, **opts): try: with existing.existingHby(name=name, base=base, bran=bran) as hby: + reger = viring.Reger(name=hby.name, db=hby.db, temp=False) + escrows = dict() - # KEL / Baser escrows + # KEL / Baser escrows - counts only if (not escrow) or escrow == "unverified-receipts": escrows["unverified-receipts"] = sum(1 for key, _ in hby.db.getUreItemIter()) @@ -59,15 +61,15 @@ def escrows(tymth, tock=0.0, **opts): if (not escrow) or escrow == "verified-receipts": escrows["verified-receipts"] = sum(1 for key, _ in hby.db.getVreItemIter()) - if (not escrow) or escrow == "partially-signed-events": - pses = list() + if (not escrow) or escrow == "out-of-order-events": + oots = list() key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in hby.db.getPseItemsNextIter(key=key): + while True: + for ekey, edig in hby.db.getOoeItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: - pses.append(eventing.loadEvent(hby.db, pre, edig)) + oots.append(eventing.loadEvent(hby.db, pre, edig)) except ValueError as e: raise e @@ -75,7 +77,7 @@ def escrows(tymth, tock=0.0, **opts): break key = ekey # setup next while iteration, with key after ekey - escrows["partially-signed-events"] = pses + escrows["out-of-order-events"] = oots if (not escrow) or escrow == "partially-witnessed-events": pwes = list() @@ -95,18 +97,15 @@ def escrows(tymth, tock=0.0, **opts): escrows["partially-witnessed-events"] = pwes - if (not escrow) or escrow == "unverified-event-indexed-couples": - escrows["unverified-event-indexed-couples"] = sum(1 for key, _ in hby.db.getUweItemIter()) - - if (not escrow) or escrow == "out-of-order-events": - oots = list() + if (not escrow) or escrow == "partially-signed-events": + pses = list() key = ekey = b'' # both start same. when not same means escrows found - while True: - for ekey, edig in hby.db.getOoeItemIter(key=key): + while True: # break when done + for ekey, edig in hby.db.getPseItemIter(key=key): pre, sn = dbing.splitSnKey(ekey) # get pre and sn from escrow item try: - oots.append(eventing.loadEvent(hby.db, pre, edig)) + pses.append(eventing.loadEvent(hby.db, pre, edig)) except ValueError as e: raise e @@ -114,7 +113,7 @@ def escrows(tymth, tock=0.0, **opts): break key = ekey # setup next while iteration, with key after ekey - escrows["out-of-order-events"] = oots + escrows["partially-signed-events"] = pses if (not escrow) or escrow == "likely-duplicitous-events": ldes = list() @@ -134,11 +133,14 @@ def escrows(tymth, tock=0.0, **opts): escrows["likely-duplicitous-events"] = ldes + if (not escrow) or escrow == "unverified-event-indexed-couples": + escrows["unverified-event-indexed-couples"] = sum(1 for key, _ in hby.db.getUweItemIter()) + if (not escrow) or escrow == "query-not-found": - escrows["query-not-found"] = sum(1 for key, _ in hby.db.getQnfItemsNextIter()) + escrows["query-not-found"] = sum(1 for key, _ in hby.db.qnfs.getItemIter()) if (not escrow) or escrow == "partially-delegated-events": - escrows["partially-delegated-events"] = sum(1 for key, _ in hby.db.getPdeItemsNextIter()) + escrows["partially-delegated-events"] = sum(1 for key, _ in hby.db.pdes.getItemIter()) if (not escrow) or escrow == "reply": escrows["reply"] = sum(1 for key, _ in hby.db.rpes.getItemIter()) @@ -146,35 +148,34 @@ def escrows(tymth, tock=0.0, **opts): if (not escrow) or escrow == "failed-oobi": escrows["failed-oobi"] = sum(1 for key, _ in hby.db.eoobi.getItemIter()) - if (not escrow) or escrow == 'group-partial-witness': + if (not escrow) or escrow == "group-partial-witness": escrows["group-partial-witness"] = sum(1 for key, _ in hby.db.gpwe.getItemIter()) - if (not escrow) or escrow == 'group-delegate': + if (not escrow) or escrow == "group-delegate": escrows["group-delegate"] = sum(1 for key, _ in hby.db.gdee.getItemIter()) - if (not escrow) or escrow == 'delegated-partial-witness': + if (not escrow) or escrow == "delegated-partial-witness": escrows["delegated-partial-witness"] = sum(1 for key, _ in hby.db.dpwe.getItemIter()) - if (not escrow) or escrow == 'group-partial-signed': + if (not escrow) or escrow == "group-partial-signed": escrows["group-partial-signed"] = sum(1 for key, _ in hby.db.gpse.getItemIter()) - if (not escrow) or escrow == 'exchange-partial-signed': + if (not escrow) or escrow == "exchange-partial-signed": escrows["exchange-partial-signed"] = sum(1 for key, _ in hby.db.epse.getItemIter()) - if (not escrow) or escrow == 'delegated-unanchored': + if (not escrow) or escrow == "delegated-unanchored": escrows["delegated-unanchored"] = sum(1 for key, _ in hby.db.dune.getItemIter()) # TEL / Reger escrows - reger = viring.Reger(name=hby.name, db=hby.db, temp=False) - if (not escrow) or escrow == 'tel-out-of-order': + if (not escrow) or escrow == "tel-out-of-order": escrows["tel-out-of-order"] = sum(1 for key, _ in reger.getOotItemIter()) - if (not escrow) or escrow == 'tel-partially-witnessed': - escrows["tel-partially-witnessed"] = sum(1 for key, _ in reger.getAllItemIter(reger.twes)) + if (not escrow) or escrow == "tel-partially-witnessed": + escrows["tel-partially-witnessed"] = sum(1 for key, _ in reger.getTweItemIter()) - if (not escrow) or escrow == 'tel-anchorless': - escrows["tel-anchorless"] = sum(1 for key, _ in reger.getAllItemIter(reger.taes)) + if (not escrow) or escrow == "tel-anchorless": + escrows["tel-anchorless"] = sum(1 for key, _ in reger.getTaeItemIter()) if (not escrow) or escrow == "missing-registry-escrow": creds = list() @@ -200,18 +201,33 @@ def escrows(tymth, tock=0.0, **opts): escrows["missing-schema-escrow"] = creds - if (not escrow) or escrow == 'tel-missing-signature': + if (not escrow) or escrow == "tel-missing-signature": escrows["tel-missing-signature"] = sum(1 for key, _ in reger.cmse.getItemIter()) - if (not escrow) or escrow == 'tel-partial-witness-escrow': + if (not escrow) or escrow == "tel-partial-witness-escrow": escrows["tel-partial-witness-escrow"] = sum(1 for key, _ in reger.tpwe.getItemIter()) - if (not escrow) or escrow == 'tel-multisig': + if (not escrow) or escrow == "tel-multisig": escrows["tel-multisig"] = sum(1 for key, _ in reger.tmse.getItemIter()) - if (not escrow) or escrow == 'tel-event-dissemination': + if (not escrow) or escrow == "tel-event-dissemination": escrows["tel-event-dissemination"] = sum(1 for key, _ in reger.tede.getItemIter()) + if (not escrow) or escrow == "registry-missing-anchor": + escrows["registry-missing-anchor"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", ""))) + + if (not escrow) or escrow == "registry-out-of-order": + escrows["registry-out-of-order"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", ""))) + + if (not escrow) or escrow == "credential-missing-registry": + escrows["credential-missing-registry"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", ""))) + + if (not escrow) or escrow == "credential-missing-anchor": + escrows["credential-missing-anchor"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", ""))) + + if (not escrow) or escrow == "credential-out-of-order": + escrows["credential-out-of-order"] = sum(1 for key, _ in reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", ""))) + print(json.dumps(escrows, indent=2)) except ConfigurationError as e: diff --git a/src/keri/app/cli/commands/event.py b/src/keri/app/cli/commands/event.py new file mode 100644 index 000000000..647df2a38 --- /dev/null +++ b/src/keri/app/cli/commands/event.py @@ -0,0 +1,84 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import json + +from keri import help +from hio.base import doing + +from keri.app.cli.common import existing +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Print an event from an AID, or specific values from an event (defaults to latest event).') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None, + required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--said', '-S', help='Print the SAID of the event in question', action="store_true") +parser.add_argument('--sn', '-s', help='Print the decimal value of the sequence number of the event in question', + action="store_true") +parser.add_argument('--snh', help='Print the decimal value of the sequence number of the event in question', + action="store_true") +parser.add_argument('--raw', '-r', help='Print the raw, signed value of the event', action="store_true") +parser.add_argument('--json', '-j', help='Pretty print the JSON of the event.', action="store_true") +parser.add_argument('--seal', help='Print an anchorable seal of the event in question.', action="store_true") + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(event, **kwa)] + + +def event(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + if hab is None: + print(f"{alias} is not a valid identifier") + + if args.said: + print(hab.kever.serder.said) + + if args.sn: + print(hab.kever.sn) + + if args.snh: + print(hab.kever.serder.snh) + + if args.raw: + print(hab.kever.serder.raw.decode("utf-8")) + + if args.json: + print(hab.kever.serder.pretty()) + + if args.seal: + seal = dict(i=hab.pre, s=hab.kever.serder.snh, d=hab.kever.serder.said) + print(json.dumps(seal)) + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/export.py b/src/keri/app/cli/commands/export.py index a2ef4d2b7..3080814cc 100644 --- a/src/keri/app/cli/commands/export.py +++ b/src/keri/app/cli/commands/export.py @@ -11,11 +11,11 @@ from hio.base import doing from keri.app.cli.common import existing -from keri.core import serdering +from keri.core import coring, serdering logger = help.ogler.getLogger() -parser = argparse.ArgumentParser(description='List credentials and check mailboxes for any newly issued credentials') +parser = argparse.ArgumentParser(description='Export key events in CESR stream format') parser.set_defaults(handler=lambda args: export(args), transferable=True) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) @@ -23,7 +23,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--files", help="export artifacts to individual files keyed off of AIDs or SAIDS, default is " "stdout", action="store_true") @@ -57,7 +57,7 @@ def __init__(self, name, alias, base, bran, ends, files): super(ExportDoer, self).__init__(doers=doers) - def exportDo(self, tymth, tock=0.0): + def exportDo(self, tymth, tock=0.0, **kwa): """ Export credential from store and any related material Parameters: @@ -106,11 +106,10 @@ def outputEnds(self, pre): f = open(f"{pre}-ends.cesr", "w") msgs = self.hab.replyToOobi(aid=pre, role="controller") - for msg in msgs: - if f is not None: - f.write(msg.decode("utf-8")) - else: - serder = serdering.SerderKERI(raw=msg) - atc = msg[serder.size:] - sys.stdout.write(serder.raw.decode("utf-8")) - sys.stdout.write(atc.decode("utf-8")) + if f is not None: + f.write(msgs.decode("utf-8")) + else: + serder = serdering.SerderKERI(raw=msgs) + atc = msgs[serder.size:] + sys.stdout.write(serder.raw.decode("utf-8")) + sys.stdout.write(atc.decode("utf-8")) diff --git a/src/keri/app/cli/commands/incept.py b/src/keri/app/cli/commands/incept.py index 87977ebd5..47364d9ff 100644 --- a/src/keri/app/cli/commands/incept.py +++ b/src/keri/app/cli/commands/incept.py @@ -29,7 +29,7 @@ parser.add_argument('--file', '-f', help='Filename to use to create the identifier', default="", required=False) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) @@ -117,6 +117,8 @@ def mergeArgsWithFile(args): incept_opts.estOnly = args.est_only if args.data is not None: incept_opts.data = config.parseData(args.data) + if args.delpre is not None: + incept_opts.delpre = args.delpre return incept_opts @@ -136,20 +138,19 @@ def __init__(self, name, base, alias, bran, endpoint, proxy=None, cnfg=None, **k reopen=True, clear=False) self.endpoint = endpoint - self.proxy = proxy - hby = existing.setupHby(name=name, base=base, bran=bran, cf=cf) - self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer - self.swain = delegating.Sealer(hby=hby) - self.postman = forwarding.Poster(hby=hby) - self.mbx = indirecting.MailboxDirector(hby=hby, topics=['/receipt', "/replay", "/reply"]) + self.hby = existing.setupHby(name=name, base=base, bran=bran, cf=cf) + self.proxy = self.hby.habByName(proxy) if proxy is not None else None + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer + self.swain = delegating.Anchorer(hby=self.hby, proxy=self.proxy) + self.postman = forwarding.Poster(hby=self.hby) + self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.postman, self.mbx, self.swain, doing.doify(self.inceptDo)] self.inits = kwa self.alias = alias - self.hby = hby super(InceptDoer, self).__init__(doers=doers) - def inceptDo(self, tymth, tock=0.0): + def inceptDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -168,8 +169,8 @@ def inceptDo(self, tymth, tock=0.0): receiptor = agenting.Receiptor(hby=self.hby) self.extend([witDoer, receiptor]) - if hab.kever.delegator: - self.swain.delegation(pre=hab.pre, sn=0, proxy=self.hby.habByName(self.proxy)) + if hab.kever.delpre: + self.swain.delegation(pre=hab.pre, sn=0) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): yield self.tock @@ -183,8 +184,12 @@ def inceptDo(self, tymth, tock=0.0): while not witDoer.cues: _ = yield self.tock - if hab.kever.delegator: - yield from self.postman.sendEvent(hab=hab, fn=hab.kever.sn) + if hab.kever.delpre: + if self.proxy is not None: + sender = self.proxy + else: + sender = hab + yield from self.postman.sendEventToDelegator(hab=hab, sender=sender, fn=hab.kever.sn) print(f'Prefix {hab.pre}') for idx, verfer in enumerate(hab.kever.verfers): diff --git a/src/keri/app/cli/commands/init.py b/src/keri/app/cli/commands/init.py index 04a1ae45c..60bcf0516 100644 --- a/src/keri/app/cli/commands/init.py +++ b/src/keri/app/cli/commands/init.py @@ -5,13 +5,17 @@ """ import argparse import getpass +import os +import sys from keri import help from hio.base import doing import keri.app.oobiing -from keri.app import habbing, configing, oobiing, connecting +from keri import kering +from keri.app import habbing, configing, oobiing from keri.app.keeping import Algos +from keri.db import basing from keri.kering import ConfigurationError from keri.vdr import credentialing @@ -48,7 +52,7 @@ def handler(args): # Parameters for Manager creation # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) parser.add_argument('--nopasscode', help='create an unencrypted keystore', action='store_true') parser.add_argument('--aeid', '-a', help='qualified base64 of non-transferable identifier prefix for authentication ' @@ -63,7 +67,7 @@ def __init__(self, args): self.args = args super(InitDoer, self).__init__(doers=[doing.doify(self.initialize)]) - def initialize(self, tymth, tock=0.0): + def initialize(self, tymth, tock=0.0, **kwa): # enter context self.wind(tymth) @@ -82,7 +86,7 @@ def initialize(self, tymth, tock=0.0): configDir = args.configDir if not args.nopasscode and not bran: - print("Creating encrypted keystore, please enter your 22 character passcode:") + print("Creating encrypted keystore, please enter your 21 character passcode:") while True: bran = getpass.getpass("Passcode: ") retry = getpass.getpass("Re-enter passcode: ") diff --git a/src/keri/app/cli/commands/interact.py b/src/keri/app/cli/commands/interact.py index e768d2994..138143511 100644 --- a/src/keri/app/cli/commands/interact.py +++ b/src/keri/app/cli/commands/interact.py @@ -5,11 +5,11 @@ """ import argparse import json -from ordered_set import OrderedSet as oset from hio.base import doing from keri import kering +from keri.help import helping from ..common import existing from ... import habbing, agenting, indirecting @@ -19,9 +19,16 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--data', '-d', help='Anchor data, \'@\' allowed', default=None, action="store", required=False) +parser.add_argument("--receipt-endpoint", help="Attempt to connect to witness receipt endpoint for witness receipts.", + dest="endpoint", action='store_true') +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') +parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', + default=[], action="append", required=False) +parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False) def interact(args): @@ -52,7 +59,8 @@ def interact(args): else: data = None - ixnDoer = InteractDoer(name=name, base=base, alias=alias, bran=bran, data=data) + ixnDoer = InteractDoer(name=name, base=base, alias=alias, bran=bran, data=data, authenticate=args.authenticate, + endpoint=args.endpoint, codes=args.code, codeTime=args.code_time) return [ixnDoer] @@ -63,7 +71,8 @@ class InteractDoer(doing.DoDoer): to all appropriate witnesses """ - def __init__(self, name, base, bran, alias, data: list = None): + def __init__(self, name, base, bran, alias, data: list = None, endpoint=False, authenticate=False, + codes=None, codeTime=None): """ Returns DoDoer with all registered Doers needed to perform interaction event. @@ -75,6 +84,10 @@ def __init__(self, name, base, bran, alias, data: list = None): self.alias = alias self.data = data + self.endpoint = endpoint + self.authenticate = authenticate + self.codes = codes if codes is not None else [] + self.codeTime = codeTime self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer @@ -96,20 +109,44 @@ def interactDo(self, tymth, tock=0.0, **opts): hab = self.hby.habByName(name=self.alias) hab.interact(data=self.data) - witDoer = agenting.WitnessReceiptor(hby=self.hby) - self.extend(doers=[witDoer]) + auths = {} + if self.authenticate: + codeTime = helping.fromIso8601(self.codeTime) if self.codeTime is not None else helping.nowIso8601() + for arg in self.codes: + (wit, code) = arg.split(":") + auths[wit] = f"{code}#{codeTime}" - if hab.kever.wits: - witDoer.msgs.append(dict(pre=hab.pre)) - while not witDoer.cues: - _ = yield self.tock + for wit in hab.kever.wits: + if wit in auths: + continue + code = input(f"Enter code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + + if self.endpoint: + receiptor = agenting.Receiptor(hby=self.hby) + self.extend([receiptor]) + + yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) + self.remove([receiptor]) + + else: + + witDoer = agenting.WitnessReceiptor(hby=self.hby, auths=auths) + self.extend(doers=[witDoer]) + + if hab.kever.wits: + witDoer.msgs.append(dict(pre=hab.pre)) + while not witDoer.cues: + _ = yield self.tock + + self.remove([witDoer]) print(f'Prefix {hab.pre}') print(f'New Sequence No. {hab.kever.sn}') for idx, verfer in enumerate(hab.kever.verfers): - print(f'\tPublic key {idx+1}: {verfer.qb64}') + print(f'\tPublic key {idx + 1}: {verfer.qb64}') - toRemove = [self.hbyDoer, witDoer, self.mbx] + toRemove = [self.hbyDoer, self.mbx] self.remove(toRemove) return diff --git a/src/keri/app/cli/commands/introduce.py b/src/keri/app/cli/commands/introduce.py new file mode 100644 index 000000000..f5f5a2ae7 --- /dev/null +++ b/src/keri/app/cli/commands/introduce.py @@ -0,0 +1,137 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from hio.base import doing + +from keri.core import serdering +from ..common import existing +from ... import habbing, connecting, forwarding + +parser = argparse.ArgumentParser(description='Send an rpy /introduce message to recipient with OOBI') +parser.set_defaults(handler=lambda args: introduce(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--recipient', help='alias or contact to send the introduction to', required=True) +parser.add_argument('--introducee', help='alias or contact of the OOBI to send', required=True) +parser.add_argument("--role", "-r", help="role of oobi to send", required=True) + + +def introduce(args): + """ + Performs a rotation of the identifier of the environment represented by the provided name parameter + + args (parseargs): Command line argument + + """ + name = args.name + alias = args.alias + base = args.base + bran = args.bran + introducee = args.introducee + recipient = args.recipient + role = args.role + + ixnDoer = IntroduceDoer(name=name, base=base, alias=alias, bran=bran, introducee=introducee, recipient=recipient, + role=role,) + + return [ixnDoer] + + +class IntroduceDoer(doing.DoDoer): + """ + + DoDoer that launches Doers needed to introduce one controller of an AID to another + + """ + + def __init__(self, name, base, bran, alias, introducee, recipient, role): + """ + Returns DoDoer with all registered Doers needed to perform an introduction. + + Parameters: + name is human readable str of identifier + """ + + self.alias = alias + self.introducee = introducee + self.recipient = recipient + self.role = role + + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer + self.org = connecting.Organizer(hby=self.hby) + self.postman = forwarding.Poster(hby=self.hby) + doers = [self.hbyDoer, self.postman, doing.doify(self.introduceDo)] + + super(IntroduceDoer, self).__init__(doers=doers) + + def introduceDo(self, tymth, tock=0.0, **opts): + """ + Returns: doifiable Doist compatible generator method + Usage: + add result of doify on this method to doers list + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + hab = self.hby.habByName(name=self.alias) + + if self.recipient in self.hby.kevers: + recp = self.recipient + else: + recp = self.org.find("alias", self.recipient) + if len(recp) != 1: + raise ValueError(f"invalid recipient {self.recipient}") + recp = recp[0]['id'] + + if (ihab := self.hby.habByName(self.introducee)) is not None: + introducee = ihab.pre + elif self.introducee in self.hby.habs: + introducee = self.introducee + elif self.introducee in self.hby.kevers: + introducee = self.introducee + else: + introducee = None + results = self.org.find("alias", self.introducee) + for result in results: + if result["alias"] == self.introducee: + introducee = result['id'] + if not introducee: + raise ValueError(f"invalid introducee {self.introducee}") + + oobi = None + for (key,), obr in self.hby.db.roobi.getItemIter(): + if obr.cid == introducee and obr.role == self.role: + oobi = key + + if oobi is None: + raise ValueError(f"Unable to find OOBI with role {self.role} for {introducee}") + + msg = hab.reply(route="/introduce", data=dict( + cid=hab.pre, + oobi=oobi + )) + + serder = serdering.SerderKERI(raw=msg) + atc = msg[serder.size:] + self.postman.send(src=hab.pre, dest=recp, topic="credential", serder=serder, + attachment=atc) + + while not len(self.postman.cues) == 1: + yield self.tock + + print(f"Introduction with OOBI {oobi} sent for {self.introducee} sent to {self.recipient}") + + toRemove = [self.hbyDoer, self.postman] + self.remove(toRemove) + + return diff --git a/src/keri/app/cli/commands/ipex/admit.py b/src/keri/app/cli/commands/ipex/admit.py index 842533805..64f496185 100644 --- a/src/keri/app/cli/commands/ipex/admit.py +++ b/src/keri/app/cli/commands/ipex/admit.py @@ -25,7 +25,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--said", "-s", help="SAID of the exn grant message to admit", required=True) @@ -77,7 +77,7 @@ def __init__(self, name, alias, base, bran, said, message, timestamp ): self.toRemove = [mbx, self.witq] super(AdmitDoer, self).__init__(doers=self.toRemove + [doing.doify(self.admitDo)]) - def admitDo(self, tymth, tock=0.0): + def admitDo(self, tymth, tock=0.0, **kwa): """ Admit credential by accepting into database and sending /ipex/admit exn message Parameters: @@ -108,7 +108,7 @@ def admitDo(self, tymth, tock=0.0): # Lets get the latest KEL and Registry if needed self.witq.query(src=self.hab.pre, pre=issr) if "ri" in acdc: - self.witq.telquery(src=self.hab.pre, wits=self.hab.kevers[issr].wits, ri=acdc["ri"], i=acdc["d"]) + self.witq.telquery(src=self.hab.pre, pre=issr, wits=self.hab.kevers[issr].wits, ri=acdc["ri"], i=acdc["d"]) for label in ("anc", "iss", "acdc"): ked = embeds[label] diff --git a/src/keri/app/cli/commands/ipex/agree.py b/src/keri/app/cli/commands/ipex/agree.py index 5e04b49ee..930ce306b 100644 --- a/src/keri/app/cli/commands/ipex/agree.py +++ b/src/keri/app/cli/commands/ipex/agree.py @@ -7,7 +7,7 @@ from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Reply to IPEX offer message acknowledged willingness to accept offered ' 'credential') @@ -18,9 +18,9 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/app/cli/commands/ipex/apply.py b/src/keri/app/cli/commands/ipex/apply.py index b5d3d8ef9..3755b01de 100644 --- a/src/keri/app/cli/commands/ipex/apply.py +++ b/src/keri/app/cli/commands/ipex/apply.py @@ -7,7 +7,7 @@ from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Request a credential from another party by initiating an IPEX exchange') parser.set_defaults(handler=lambda args: handler(args)) @@ -17,9 +17,9 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/app/cli/commands/ipex/grant.py b/src/keri/app/cli/commands/ipex/grant.py index 38fc0d33b..74fdb68b3 100644 --- a/src/keri/app/cli/commands/ipex/grant.py +++ b/src/keri/app/cli/commands/ipex/grant.py @@ -23,7 +23,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--recipient", "-r", help="alias or qb64 identifier prefix of the self.recp of " @@ -71,7 +71,7 @@ def __init__(self, name, alias, base, bran, said, recp, message, timestamp): self.toRemove = [mbx] super(GrantDoer, self).__init__(doers=self.toRemove + [doing.doify(self.grantDo)]) - def grantDo(self, tymth, tock=0.0): + def grantDo(self, tymth, tock=0.0, **kwa): """ Grant credential by creating /ipex/grant exn message Parameters: @@ -108,11 +108,11 @@ def grantDo(self, tymth, tock=0.0): iss = self.rgy.reger.cloneTvtAt(creder.said) - iserder = serdering.SerderKERI(raw=bytes(iss)) # coring.Serder(raw=bytes(iss)) + iserder = serdering.SerderKERI(raw=bytes(iss)) seqner = coring.Seqner(sn=iserder.sn) - serder = self.hby.db.findAnchoringSealEvent(creder.sad['i'], - seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) + serder = self.hby.db.fetchLastSealingEventByEventSeal(creder.sad['i'], + seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said)) anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc, diff --git a/src/keri/app/cli/commands/ipex/join.py b/src/keri/app/cli/commands/ipex/join.py index 0f8401514..122142145 100644 --- a/src/keri/app/cli/commands/ipex/join.py +++ b/src/keri/app/cli/commands/ipex/join.py @@ -23,7 +23,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") @@ -90,7 +90,7 @@ def __init__(self, name, base, bran, auto=False): self.auto = auto super(JoinDoer, self).__init__(doers=doers) - def joinDo(self, tymth, tock=0.0): + def joinDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/ipex/list.py b/src/keri/app/cli/commands/ipex/list.py index 89449d87f..156684943 100644 --- a/src/keri/app/cli/commands/ipex/list.py +++ b/src/keri/app/cli/commands/ipex/list.py @@ -8,6 +8,7 @@ import argparse import datetime import os +import json import sys from keri import help @@ -33,7 +34,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print full JSON of all credentials", action="store_true") @@ -89,7 +90,7 @@ def __init__(self, name, alias, base, bran, poll=False, verbose=False, said=Fals super(ListDoer, self).__init__(doers=self.doers + [doing.doify(self.listDo)]) - def listDo(self, tymth, tock=0.0): + def listDo(self, tymth, tock=0.0, **kwa): """ Check for any credential messages in mailboxes and list all held credentials Parameters: @@ -199,6 +200,12 @@ def grant(self, exn): if response is not None: print(f" Response: {responseType} ({response.qb64})") + if self.verbose: + bsad = json.dumps(sad, indent=2) + print(" Full Credential:") + for line in bsad.splitlines(): + print(f"\t{line}") + def apply(self, note, exn, pathed): pass diff --git a/src/keri/app/cli/commands/ipex/offer.py b/src/keri/app/cli/commands/ipex/offer.py index a297b234f..cf98ff1bb 100644 --- a/src/keri/app/cli/commands/ipex/offer.py +++ b/src/keri/app/cli/commands/ipex/offer.py @@ -7,7 +7,7 @@ from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Reply to IPEX apply message or initiate an IPEX exchange with an offer' ' for a credential with certain characteristics') @@ -18,9 +18,9 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/app/cli/commands/ipex/spurn.py b/src/keri/app/cli/commands/ipex/spurn.py index 3d4ecebe8..573b99d3b 100644 --- a/src/keri/app/cli/commands/ipex/spurn.py +++ b/src/keri/app/cli/commands/ipex/spurn.py @@ -26,7 +26,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--said", "-s", help="SAID of the exn IPEX message to spurn", required=True) @@ -74,7 +74,7 @@ def __init__(self, name, alias, base, bran, said, message): self.toRemove = [mbx] super(SpurnDoer, self).__init__(doers=self.toRemove + [doing.doify(self.spurnDo)]) - def spurnDo(self, tymth, tock=0.0): + def spurnDo(self, tymth, tock=0.0, **kwa): """ Sprun any IPEX message Parameters: diff --git a/src/keri/app/cli/commands/kevers.py b/src/keri/app/cli/commands/kevers.py index 3cc6eb905..4aa0edb13 100644 --- a/src/keri/app/cli/commands/kevers.py +++ b/src/keri/app/cli/commands/kevers.py @@ -13,19 +13,19 @@ from keri.app import indirecting from keri.app.cli.common import displaying, existing -from keri.core import serdering +from keri.core import coring, serdering from keri.help import helping logger = help.ogler.getLogger() -parser = argparse.ArgumentParser(description='Initialize a prefix') +parser = argparse.ArgumentParser(description='Poll events at controller for prefix') parser.set_defaults(handler=lambda args: handler(args), transferable=True) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--prefix', help='qb64 identifier prefix to display', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--poll", "-P", help="Poll mailboxes for any events", action="store_true") diff --git a/src/keri/app/cli/commands/list.py b/src/keri/app/cli/commands/list.py index 4d36d622f..4cb8d5567 100644 --- a/src/keri/app/cli/commands/list.py +++ b/src/keri/app/cli/commands/list.py @@ -10,6 +10,7 @@ from hio.base import doing from keri.app.cli.common import existing +from keri.kering import ConfigurationError logger = help.ogler.getLogger() @@ -19,7 +20,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/local/watch.py b/src/keri/app/cli/commands/local/watch.py index 3558a0f44..eae8ccbb0 100644 --- a/src/keri/app/cli/commands/local/watch.py +++ b/src/keri/app/cli/commands/local/watch.py @@ -7,13 +7,14 @@ import random import sys import time -from collections import namedtuple from keri import help from hio.base import doing +from hio.help import decking from keri.app import agenting, indirecting, habbing, forwarding from keri.app.cli.common import existing, terming from keri.app.habbing import GroupHab +from keri.app.watching import States, diffState logger = help.ogler.getLogger() @@ -25,22 +26,11 @@ required=False, default="") # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) -Stateage = namedtuple("Stateage", 'even ahead behind duplicitous') - -States = Stateage(even="even", ahead="ahead", behind="behind", duplicitous="duplicitous") - - -class WitnessState: - wit: str - state: Stateage - sn: int - dig: str - def watch(args): name = args.name @@ -55,7 +45,7 @@ def __init__(self, name, base, bran, **kwa): doers = [] self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer - self.cues = help.decking.Deck() + self.cues = decking.Deck() self.mbd = indirecting.MailboxDirector(hby=self.hby, topics=["/replay", "/receipt", "/reply"]) self.postman = forwarding.Poster(hby=self.hby) @@ -134,7 +124,7 @@ def watchDo(self, tymth, tock=0.0, **opts): mystate = hab.kever.state() witstate = hab.db.ksns.get((saider.qb64,)) - states.append(self.diffState(wit, mystate, witstate)) + states.append(diffState(wit, mystate, witstate)) # First check for any duplicity, if so get out of here dups = [state for state in states if state.state == States.duplicitous] @@ -212,31 +202,3 @@ def cueDo(self, tymth, tock=0.0, **opts): yield self.tock yield self.tock - - @staticmethod - def diffState(wit, preksn, witksn): - - witstate = WitnessState() - witstate.wit = wit - mysn = int(preksn.s, 16) - mydig = preksn.d - witstate.sn = int(witksn.f, 16) - witstate.dig = witksn.d - - # At the same sequence number, check the DIGs - if mysn == witstate.sn: - if mydig == witstate.dig: - witstate.state = States.even - else: - witstate.state = States.duplicitous - - # This witness is behind and will need to be caught up. - elif mysn > witstate.sn: - witstate.state = States.behind - - # mysn < witstate.sn - We are behind this witness (multisig or restore situation). - # Must ensure that controller approves this event or a recovery rotation is needed - else: - witstate.state = States.ahead - - return witstate diff --git a/keri/cf/demo-witness-oobis.json b/src/keri/app/cli/commands/location/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from keri/cf/demo-witness-oobis.json rename to src/keri/app/cli/commands/location/__init__.py diff --git a/src/keri/app/cli/commands/location/add.py b/src/keri/app/cli/commands/location/add.py new file mode 100644 index 000000000..52c1e173f --- /dev/null +++ b/src/keri/app/cli/commands/location/add.py @@ -0,0 +1,123 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +from urllib.parse import urlparse + +from keri import help +from hio.base import doing + +from keri import kering +from keri.app import habbing, grouping, indirecting, forwarding +from keri.app.agenting import WitnessPublisher +from keri.app.cli.common import existing +from keri.app.notifying import Notifier +from keri.core import parsing +from keri.peer import exchanging + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add new endpoint location record.') +parser.set_defaults(handler=lambda args: add_loc(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--url", "-u", help="Location URL", + required=True) +parser.add_argument("--eid", "-e", help="qualified base64 of AID to associate a location with, defaults to alias aid ", + required=False, default=None) +parser.add_argument("--time", help="timestamp for the end auth", required=False, default=None) + + +def add_loc(args): + """ Command line tool for adding location scheme records + + """ + ld = LocationDoer(name=args.name, + base=args.base, + alias=args.alias, + bran=args.bran, + url=args.url, + eid=args.eid, + timestamp=args.time) + return [ld] + + +class LocationDoer(doing.DoDoer): + + def __init__(self, name, base, alias, bran, url, eid, timestamp=None): + self.url = url + self.eid = eid + self.timestamp = timestamp + + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.witpub = WitnessPublisher(hby=self.hby) + self.postman = forwarding.Poster(hby=self.hby) + notifier = Notifier(self.hby) + mux = grouping.Multiplexor(self.hby, notifier=notifier) + exc = exchanging.Exchanger(hby=self.hby, handlers=[]) + grouping.loadHandlers(exc, mux) + + mbx = indirecting.MailboxDirector(hby=self.hby, topics=["/receipt", "/multisig", "/replay"], exc=exc) + + if self.hab is None: + raise kering.ConfigurationError(f"unknown alias={alias}") + + self.toRemove = [self.witpub, self.postman, mbx] + + super(LocationDoer, self).__init__(doers=self.toRemove + [doing.doify(self.roleDo)]) + + def roleDo(self, tymth, tock=0.0, **kwa): + """ Export any end reply messages previous saved for the provided AID + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + up = urlparse(self.url) + eid = self.eid if self.eid is not None else self.hab.pre + + msg = self.hab.makeLocScheme(url=self.url, eid=eid, scheme=up.scheme) + parsing.Parser().parse(ims=bytes(msg), kvy=self.hab.kvy, rvy=self.hab.rvy) + + if isinstance(self.hab, habbing.GroupHab): + smids = self.hab.db.signingMembers(pre=self.hab.pre) + smids.remove(self.hab.mhab.pre) + + for recp in smids: # this goes to other participants only as a signaling mechanism + exn, atc = grouping.multisigRpyExn(ghab=self.hab, rpy=msg) + self.postman.send(src=self.hab.mhab.pre, + dest=recp, + topic="multisig", + serder=exn, + attachment=atc) + + while not self.hab.loadLocScheme(scheme=up.scheme, eid=eid): + yield self.tock + + self.witpub.msgs.append(dict(pre=self.hab.pre, msg=bytes(msg))) + + while not self.witpub.cues: + yield self.tock + + print(f"Location {self.url} added for aid {eid} with scheme {up.scheme}") + + self.remove(self.toRemove) + return diff --git a/src/keri/app/cli/commands/mailbox/add.py b/src/keri/app/cli/commands/mailbox/add.py new file mode 100644 index 000000000..543ab2e08 --- /dev/null +++ b/src/keri/app/cli/commands/mailbox/add.py @@ -0,0 +1,135 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from keri import help +from hio.base import doing +from hio.help import Hict + +from keri import kering +from keri.app import connecting, habbing, forwarding +from keri.app.agenting import httpClient, WitnessPublisher +from keri.app.cli.common import existing +from keri.core import serdering + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add mailbox role') +parser.set_defaults(handler=lambda args: add(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--mailbox", '-w', help="the mailbox AID or alias to add", required=True) + + +def add(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + ed = AddDoer(name=args.name, + alias=args.alias, + base=args.base, + bran=args.bran, + mailbox=args.mailbox) + return [ed] + + +class AddDoer(doing.DoDoer): + + def __init__(self, name, alias, base, bran, mailbox): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.org = connecting.Organizer(hby=self.hby) + self.witpub = WitnessPublisher(hby=self.hby) + + if mailbox in self.hby.kevers: + mbx = mailbox + else: + mbx = self.org.find("alias", mailbox) + if len(mbx) != 1: + raise ValueError(f"invalid mailbox {mailbox}") + mbx = mbx[0]['id'] + + if not mbx: + raise ValueError(f"unknown mailbox {mailbox}") + + self.mailbox = mbx + + doers = [doing.doify(self.addDo), self.witpub] + + super(AddDoer, self).__init__(doers=doers) + + def addDo(self, tymth, tock=0.0, **kwa): + """ Grant credential by creating /ipex/grant exn message + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + if isinstance(self.hab, habbing.GroupHab): + raise ValueError("watchers for multisig AIDs not currently supported") + + kel = self.hab.replay() + data = dict(cid=self.hab.pre, + role=kering.Roles.mailbox, + eid=self.mailbox) + + route = "/end/role/add" + msg = self.hab.reply(route=route, data=data) + self.hab.psr.parseOne(ims=(bytes(msg))) # make copy to preserve + + fargs = dict([("kel", kel.decode("utf-8")), + ("rpy", msg.decode("utf-8"))]) + + headers = (Hict([ + ("Content-Type", "multipart/form-data"), + ])) + + client, clientDoer = httpClient(self.hab, self.mailbox) + self.extend([clientDoer]) + + client.request( + method="POST", + path=f"{client.requester.path}/mailboxes", + headers=headers, + fargs=fargs + ) + while not client.responses: + yield self.tock + + rep = client.respond() + if rep.status == 200: + msg = self.hab.replyEndRole(cid=self.hab.pre, role=kering.Roles.mailbox) + self.witpub.msgs.append(dict(pre=self.hab.pre, msg=bytes(msg))) + + while not self.witpub.cues: + yield self.tock + + print(f"Mailbox {self.mailbox} added for {self.hab.name}") + + else: + print(rep.status, rep.data) + + self.remove([clientDoer, self.witpub]) diff --git a/src/keri/app/cli/commands/mailbox/debug.py b/src/keri/app/cli/commands/mailbox/debug.py index 6cf5fe964..fcac56240 100644 --- a/src/keri/app/cli/commands/mailbox/debug.py +++ b/src/keri/app/cli/commands/mailbox/debug.py @@ -10,9 +10,11 @@ from hio.base import doing from keri import kering -from keri.app import agenting, habbing, httping -from keri.app.cli.common import existing +from keri.app import agenting, indirecting, habbing, httping +from keri.app.cli.common import displaying, existing from keri.app.habbing import GroupHab +from keri.core import coring +from keri.kering import ConfigurationError logger = help.ogler.getLogger() @@ -23,7 +25,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--witness', '-w', help='The qualified b64 AID of the witness to poll', required=True) parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") @@ -67,7 +69,7 @@ def __init__(self, name, base, alias, bran, witness, verbose): super(ReadDoer, self).__init__(doers=doers) - def readDo(self, tymth, tock=0.0): + def readDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -83,7 +85,7 @@ def readDo(self, tymth, tock=0.0): hab = self.hby.habByName(name=self.alias) topics = {"/receipt": 0, "/replay": 0, "/multisig": 0, "/credential": 0, "/delegate": 0, "/challenge": 0, - "/oobi": 0} + "/oobi": 0, "/reply": 0} try: client, clientDoer = agenting.httpClient(hab, self.witness) except kering.MissingEntryError as e: @@ -93,8 +95,11 @@ def readDo(self, tymth, tock=0.0): print("Local Index per Topic") witrec = hab.db.tops.get((hab.pre, self.witness)) - for topic in witrec.topics: - print(f" Topic {topic}: {witrec.topics[topic]}") + if witrec: + for topic in witrec.topics: + print(f" Topic {topic}: {witrec.topics[topic]}") + else: + print("\tNo local index") print() q = dict(pre=hab.pre, topics=topics) @@ -105,7 +110,7 @@ def readDo(self, tymth, tock=0.0): httping.createCESRRequest(msg, client, dest=self.witness) - while client.requests: + while client.requests or (not client.events and not client.requests): yield self.tock yield 1.0 diff --git a/src/keri/app/cli/commands/mailbox/list.py b/src/keri/app/cli/commands/mailbox/list.py index 369e749df..d03d6e443 100644 --- a/src/keri/app/cli/commands/mailbox/list.py +++ b/src/keri/app/cli/commands/mailbox/list.py @@ -6,7 +6,7 @@ """ import argparse -from hio import help +from keri import help from hio.base import doing from keri.app import connecting diff --git a/src/keri/app/cli/commands/mailbox/update.py b/src/keri/app/cli/commands/mailbox/update.py index 8ac16cd93..05d6c341d 100644 --- a/src/keri/app/cli/commands/mailbox/update.py +++ b/src/keri/app/cli/commands/mailbox/update.py @@ -27,7 +27,7 @@ parser.add_argument("--index", "-i", help="new index for topic on witness", required=True) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/migrate/list.py b/src/keri/app/cli/commands/migrate/list.py index a88360e42..0191b02b3 100644 --- a/src/keri/app/cli/commands/migrate/list.py +++ b/src/keri/app/cli/commands/migrate/list.py @@ -1,10 +1,9 @@ # -*- encoding: utf-8 -*- """ -keri.kli.commands.migrate.list module +keri.kli.commands module """ import argparse -import logging from keri import help from hio.base import doing @@ -14,6 +13,7 @@ logger = help.ogler.getLogger() + def handler(args): """ List local LMDB database migrations and their completion status @@ -53,7 +53,7 @@ def recur(self, tyme): tab.align["Name"] = "l" hby = existing.setupHby(name=self.args.name, base=self.args.base, - bran=self.args.bran if self.args.bran else None, temp=self.args.temp) + bran=self.args.bran, temp=self.args.temp) for idx, (name, dater) in enumerate(hby.db.complete()): print(name, dater) diff --git a/src/keri/app/cli/commands/migrate/run.py b/src/keri/app/cli/commands/migrate/run.py index de2a62eb4..0e9e1d52c 100644 --- a/src/keri/app/cli/commands/migrate/run.py +++ b/src/keri/app/cli/commands/migrate/run.py @@ -1,18 +1,21 @@ # -*- encoding: utf-8 -*- """ -keri.kli.commands.migrate.run module +keri.kli.commands module """ import argparse -from hio.base import doing - +import keri from keri import help +from hio.base import doing from keri import kering + +from keri.app.cli.common import existing from keri.db import basing logger = help.ogler.getLogger() + def handler(args): """ Launch KERI database migrator @@ -24,7 +27,7 @@ def handler(args): return [migrator] -parser = argparse.ArgumentParser(description='Migrates a database and keystore') +parser = argparse.ArgumentParser(description='Cleans and migrates a database and keystore up to the latest source code version') parser.set_defaults(handler=handler, transferable=True) @@ -47,21 +50,18 @@ def __init__(self, args): super(MigrateDoer, self).__init__() def recur(self, tyme): - name=self.args.name - base=self.args.base - temp=self.args.temp - hab_db = basing.Baser(name=name, - base=base, - temp=temp, - reopen=False) + db = basing.Baser(name=self.args.name, + base=self.args.base, + temp=self.args.temp, + reopen=False) try: - hab_db.reopen() - except kering.DatabaseError as ex: + db.reopen() + except kering.DatabaseError: pass - print(f"Migrating {name}...") - hab_db.migrate() - print(f"Finished migrating {name}") + print(f"Migrating {self.args.name}...") + db.migrate() + print(f"Finished migrating {self.args.name}") return True diff --git a/src/keri/app/cli/commands/migrate/show.py b/src/keri/app/cli/commands/migrate/show.py index 6bf3392c5..538d7f511 100644 --- a/src/keri/app/cli/commands/migrate/show.py +++ b/src/keri/app/cli/commands/migrate/show.py @@ -12,6 +12,7 @@ logger = help.ogler.getLogger() + def handler(args): """ Launch KERI database initialization diff --git a/src/keri/app/cli/commands/multisig/continue.py b/src/keri/app/cli/commands/multisig/continue.py index f419a7418..9ca75f360 100644 --- a/src/keri/app/cli/commands/multisig/continue.py +++ b/src/keri/app/cli/commands/multisig/continue.py @@ -23,7 +23,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran @@ -65,7 +65,7 @@ def recover(self, tymth, tock=0.0, **opts): (seqner, saider) = esc[0] src = hab.mhab.pre if isinstance(hab, GroupHab) else hab.pre anchor = dict(i=hab.pre, s=seqner.snh, d=saider.qb64) - self.witq.query(src=src, pre=hab.kever.delegator, anchor=anchor) + self.witq.query(src=src, pre=hab.kever.delpre, anchor=anchor) print(f"Checking mailboxes for any events to process") while self.hby.db.cgms.get(keys=(hab.pre, seqner.qb64)) is None: diff --git a/src/keri/app/cli/commands/multisig/incept.py b/src/keri/app/cli/commands/multisig/incept.py index 605a91040..89378404d 100644 --- a/src/keri/app/cli/commands/multisig/incept.py +++ b/src/keri/app/cli/commands/multisig/incept.py @@ -28,7 +28,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--wait", "-w", help="number of seconds to wait for other multisig events, defaults to 10", default=10) @@ -105,7 +105,7 @@ def __init__(self, name, base, alias, bran, group, wait, **kwa): super(GroupMultisigIncept, self).__init__(doers=doers) - def inceptDo(self, tymth, tock=0.0): + def inceptDo(self, tymth, tock=0.0, **kwa): """ Create or participate in an inception event for a distributed multisig identifier Parameters: @@ -156,7 +156,7 @@ def inceptDo(self, tymth, tock=0.0): serder=exn, attachment=ims) - logger.info(f"Group identifier inception initialized for {ghab.pre}") + print(f"Group identifier inception initialized for {ghab.pre}") prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=0) saider = coring.Saider(qb64=prefixer.qb64) @@ -174,8 +174,8 @@ def inceptDo(self, tymth, tock=0.0): yield self.tock - if ghab.kever.delegator: - yield from self.postman.sendEvent(hab=ghab, fn=ghab.kever.sn) + if ghab.kever.delpre: + yield from self.postman.sendEventToDelegator(hab=ghab, sender=ghab.mhab, fn=ghab.kever.sn) print() displaying.printIdentifier(self.hby, ghab.pre) diff --git a/src/keri/app/cli/commands/multisig/interact.py b/src/keri/app/cli/commands/multisig/interact.py index b5abef1e8..3892a79e9 100644 --- a/src/keri/app/cli/commands/multisig/interact.py +++ b/src/keri/app/cli/commands/multisig/interact.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--data', '-d', help='Anchor data, \'@\' allowed', default=[], action="store", required=True) parser.add_argument("--aids", "-g", help="List of other participant qb64 identifiers to include in interaction event", @@ -92,7 +92,7 @@ def __init__(self, name, alias, aids, base, bran, data): super(GroupMultisigInteract, self).__init__(doers=doers) - def interactDo(self, tymth, tock=0.0): + def interactDo(self, tymth, tock=0.0, **kwa): """ Create or participate in an interaction event for a distributed multisig identifier Parameters: diff --git a/src/keri/app/cli/commands/multisig/join.py b/src/keri/app/cli/commands/multisig/join.py index b85db6b75..6b62bc57a 100644 --- a/src/keri/app/cli/commands/multisig/join.py +++ b/src/keri/app/cli/commands/multisig/join.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--group', '-g', help='human-readable name for the multisig group identifier prefix', required=False, default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--auto", "-Y", help="auto approve any delegation request non-interactively", action="store_true") @@ -97,7 +97,7 @@ def __init__(self, name, base, bran, group, auto=False): self.auto = auto super(JoinDoer, self).__init__(doers=doers) - def joinDo(self, tymth, tock=0.0): + def joinDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -182,12 +182,12 @@ def incept(self, attrs): inits["isith"] = oicp.ked["kt"] inits["nsith"] = oicp.ked["nt"] - inits["estOnly"] = eventing.TraitCodex.EstOnly in oicp.ked["c"] - inits["DnD"] = eventing.TraitCodex.DoNotDelegate in oicp.ked["c"] + inits["estOnly"] = kering.TraitCodex.EstOnly in oicp.ked["c"] + inits["DnD"] = kering.TraitCodex.DoNotDelegate in oicp.ked["c"] inits["toad"] = oicp.ked["bt"] inits["wits"] = oicp.ked["b"] - inits["delpre"] = oicp.ked["di"] if "di" in ked else None + inits["delpre"] = oicp.ked["di"] if "di" in oicp.ked else None print() print("Group Multisig Inception proposed:") @@ -351,8 +351,8 @@ def showEvent(self, hab, mids, ked): if not thold.weighted: tab.add_row(["Signature Threshold", thold.num]) - tab.add_row(["Establishment Only", eventing.TraitCodex.EstOnly in ked["c"]]) - tab.add_row(["Do Not Delegate", eventing.TraitCodex.DoNotDelegate in ked["c"]]) + tab.add_row(["Establishment Only", eventing.TraitDex.EstOnly in ked["c"]]) + tab.add_row(["Do Not Delegate", eventing.TraitDex.DoNotDelegate in ked["c"]]) tab.add_row(["Witness Threshold", ked["bt"]]) tab.add_row(["Witnesses", "\n".join(ked["b"])]) diff --git a/src/keri/app/cli/commands/multisig/notice.py b/src/keri/app/cli/commands/multisig/notice.py index fe4ca2769..223f4da75 100644 --- a/src/keri/app/cli/commands/multisig/notice.py +++ b/src/keri/app/cli/commands/multisig/notice.py @@ -26,7 +26,7 @@ parser.add_argument("--config", "-c", help="directory override for configuration data") # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) @@ -66,7 +66,7 @@ def __init__(self, name, base, alias, bran): super(NoticeDoer, self).__init__(doers=doers) - def noticeDo(self, tymth, tock=0.0): + def noticeDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of diff --git a/src/keri/app/cli/commands/multisig/rotate.py b/src/keri/app/cli/commands/multisig/rotate.py index a1727ff84..3a6dc385b 100644 --- a/src/keri/app/cli/commands/multisig/rotate.py +++ b/src/keri/app/cli/commands/multisig/rotate.py @@ -27,7 +27,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the local identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--smids", "-s", help="List of other participant qb64 identifiers with signing authority in " "rotation event", @@ -44,7 +44,7 @@ def rotateGroupIdentifier(args): Performs a rotation on the group identifier specified as an argument. The identifier prefix of the environment represented by the name parameter must be a member of the group identifier. This command will perform a rotation of the local identifier if the sequence number of the local identifier is the same as the group identifier sequence - number. It will wait for all other members of the group to acheive the same sequence number (group + 1) and then + number. It will wait for all other members of the group to achieve the same sequence number (group + 1) and then publish the signed rotation event for the group identifier to all witnesses and wait for receipts. Parameters: @@ -203,9 +203,9 @@ def rotateDo(self, tymth, tock=0.0, **opts): prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn+1) - rot = ghab.rotate(smids=smids, rmids=rmids, isith=self.isith, nsith=self.nsith, + rot = ghab.rotate(isith=self.isith, nsith=self.nsith, toad=self.toad, cuts=list(self.cuts), adds=list(self.adds), data=self.data, - verfers=merfers, digers=migers) + verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) # Create a notification EXN message to send to the other agents @@ -233,8 +233,8 @@ def rotateDo(self, tymth, tock=0.0, **opts): yield self.tock - if ghab.kever.delegator: - yield from self.postman.sendEvent(hab=ghab, fn=ghab.kever.sn) + if ghab.kever.delpre: + yield from self.postman.sendEventToDelegator(hab=ghab, sender=ghab.mhab, fn=ghab.kever.sn) print() displaying.printIdentifier(self.hby, ghab.pre) diff --git a/src/keri/app/cli/commands/multisig/shell.py b/src/keri/app/cli/commands/multisig/shell.py index 6d0da88c2..ec4c3d093 100644 --- a/src/keri/app/cli/commands/multisig/shell.py +++ b/src/keri/app/cli/commands/multisig/shell.py @@ -15,7 +15,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--static-threshold', '-t', help='Specify this to switch to using one static number as the signing' diff --git a/src/keri/app/cli/commands/multisig/update.py b/src/keri/app/cli/commands/multisig/update.py index f528bcf18..5ef24266b 100644 --- a/src/keri/app/cli/commands/multisig/update.py +++ b/src/keri/app/cli/commands/multisig/update.py @@ -8,6 +8,7 @@ from keri import help from hio.base import doing +from hio.help import decking from keri import kering from keri.app import agenting, indirecting, habbing @@ -27,7 +28,7 @@ parser.add_argument('--witness', '-w', help='QB64 identifier of witness to query', default="", required=True) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) @@ -60,7 +61,7 @@ def __init__(self, name, alias, base, bran, wit, sn, said, **kwa): self.wit = wit self.sn = sn self.said = said - self.cues = help.decking.Deck() + self.cues = decking.Deck() self.mbd = indirecting.MailboxDirector(hby=self.hby, topics=["/replay", "/receipt", "/reply"]) self.witq = agenting.WitnessInquisitor(hby=self.hby) diff --git a/src/keri/app/cli/commands/nonce.py b/src/keri/app/cli/commands/nonce.py index 96669c816..63eee5929 100644 --- a/src/keri/app/cli/commands/nonce.py +++ b/src/keri/app/cli/commands/nonce.py @@ -8,7 +8,7 @@ import pysodium from hio.base import doing -from keri.core import coring +from keri.core import signing parser = argparse.ArgumentParser(description='Print a new random nonce') parser.set_defaults(handler=lambda args: handler(args)) @@ -18,9 +18,9 @@ def handler(_): return [doing.doify(nonce)] -def nonce(tymth, tock=0.0): +def nonce(tymth, tock=0.0, **kwa): """ nonce """ _ = (yield tock) - print(coring.randomNonce()) + print(signing.Salter().qb64) diff --git a/src/keri/app/cli/commands/notifications/list.py b/src/keri/app/cli/commands/notifications/list.py index 0788ea7c4..b4e4e1e52 100644 --- a/src/keri/app/cli/commands/notifications/list.py +++ b/src/keri/app/cli/commands/notifications/list.py @@ -7,13 +7,10 @@ import argparse import json -from keri import help from hio.base import doing - -from keri import kering -from keri.app import agenting, habbing, httping, notifying +from keri import help +from keri.app import habbing, notifying from keri.app.cli.common import existing -from keri.app.habbing import GroupHab logger = help.ogler.getLogger() @@ -63,7 +60,7 @@ def __init__(self, name, base, alias, bran, verbose): super(NotesDoer, self).__init__(doers=doers) - def readDo(self, tymth, tock=0.0): + def readDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -90,4 +87,4 @@ def readDo(self, tymth, tock=0.0): print(keys, notice.attrs.get('r', 'no route')) self.remove([self.hbyDoer,]) - return + return \ No newline at end of file diff --git a/src/keri/app/cli/commands/notifications/mark.py b/src/keri/app/cli/commands/notifications/mark.py index 0c70d9d22..9c7547acf 100644 --- a/src/keri/app/cli/commands/notifications/mark.py +++ b/src/keri/app/cli/commands/notifications/mark.py @@ -6,13 +6,10 @@ """ import argparse -from keri import help from hio.base import doing - -from keri import kering -from keri.app import agenting, habbing, httping, notifying +from keri import help +from keri.app import habbing, notifying from keri.app.cli.common import existing -from keri.app.habbing import GroupHab logger = help.ogler.getLogger() @@ -65,7 +62,7 @@ def __init__(self, name, base, alias, bran, rid, all): super(MarkDoer, self).__init__(doers=doers) - def markDo(self, tymth, tock=0.0): + def markDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -98,4 +95,4 @@ def markDo(self, tymth, tock=0.0): print("Must specify one of --rid or --all") self.remove([self.hbyDoer,]) - return + return \ No newline at end of file diff --git a/src/keri/app/cli/commands/notifications/rem.py b/src/keri/app/cli/commands/notifications/rem.py index b5f207e99..ea2c61b8d 100644 --- a/src/keri/app/cli/commands/notifications/rem.py +++ b/src/keri/app/cli/commands/notifications/rem.py @@ -6,13 +6,10 @@ """ import argparse -from keri import help from hio.base import doing - -from keri import kering -from keri.app import agenting, habbing, httping, notifying +from keri import help +from keri.app import habbing, notifying from keri.app.cli.common import existing -from keri.app.habbing import GroupHab logger = help.ogler.getLogger() @@ -65,7 +62,7 @@ def __init__(self, name, base, alias, bran, rid, all): super(RemoveDoer, self).__init__(doers=doers) - def remDoer(self, tymth, tock=0.0): + def remDoer(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -98,4 +95,4 @@ def remDoer(self, tymth, tock=0.0): print("Must specify one of --rid or --all") self.remove([self.hbyDoer,]) - return + return \ No newline at end of file diff --git a/src/keri/app/cli/commands/oobi/clean.py b/src/keri/app/cli/commands/oobi/clean.py index 2dead78d1..c09545d50 100644 --- a/src/keri/app/cli/commands/oobi/clean.py +++ b/src/keri/app/cli/commands/oobi/clean.py @@ -19,7 +19,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran def list_oobis(args): diff --git a/src/keri/app/cli/commands/oobi/generate.py b/src/keri/app/cli/commands/oobi/generate.py index 2cb0082ee..38c378ce7 100644 --- a/src/keri/app/cli/commands/oobi/generate.py +++ b/src/keri/app/cli/commands/oobi/generate.py @@ -4,7 +4,6 @@ """ import argparse -from urllib.parse import urlparse import sys from keri import help @@ -28,7 +27,7 @@ # Parameters for Manager access # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) @@ -66,18 +65,30 @@ def generate(tymth, tock=0.0, **opts): sys.exit(-1) for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise kering.ConfigurationError(f"unable to query witness {wit}, no http endpoint") - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/witness") elif role in (kering.Roles.controller,): - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: print(f"{alias} identifier {hab.pre} does not have any controller endpoints") return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - print(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") + elif role in (kering.Roles.mailbox,): + for (_, _, eid), end in hab.db.ends.getItemIter(keys=(hab.pre, kering.Roles.mailbox, )): + if not (end.allowed and end.enabled is not False): + continue + + urls = hab.fetchUrls(eid=eid, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, + scheme=kering.Schemes.https) + if not urls: + print(f"{alias} identifier {hab.pre} does not have any mailbox endpoints") + return + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + print(f"{url.rstrip("/")}/oobi/{hab.pre}/mailbox/{eid}") diff --git a/src/keri/app/cli/commands/oobi/resolve.py b/src/keri/app/cli/commands/oobi/resolve.py index b31640e19..f1376defe 100644 --- a/src/keri/app/cli/commands/oobi/resolve.py +++ b/src/keri/app/cli/commands/oobi/resolve.py @@ -12,6 +12,7 @@ from keri.app import habbing, oobiing from keri.app.cli.common import existing from keri.db import basing +from keri.end import ending from keri.help import helping logger = help.ogler.getLogger() @@ -33,7 +34,7 @@ # Parameters for Manager access # passcode => bran -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) @@ -80,7 +81,7 @@ def __init__(self, name, oobi, oobiAlias, force=False, bran=None, base=None): super(OobiDoer, self).__init__(doers=doers) - def waitDo(self, tymth, tock=0.0): + def waitDo(self, tymth, tock=0.0, **kwa): """ Waits for oobis to load Parameters: @@ -106,6 +107,11 @@ def waitDo(self, tymth, tock=0.0): yield 0.25 obr = self.obi.hby.db.roobi.get(keys=(self.oobi,)) + if self.force: + while obr.cid not in self.hby.kevers: + self.hby.kvy.processEscrows() + yield 0.25 + print(self.oobi, obr.state) self.remove([self.hbyDoer, *self.obi.doers, *self.authn.doers]) diff --git a/src/keri/app/cli/commands/passcode/generate.py b/src/keri/app/cli/commands/passcode/generate.py index 7e1d9cebb..3ac2b8c7a 100644 --- a/src/keri/app/cli/commands/passcode/generate.py +++ b/src/keri/app/cli/commands/passcode/generate.py @@ -20,7 +20,7 @@ def handler(_): return [doing.doify(salt)] -def salt(tymth, tock=0.0): +def salt(tymth, tock=0.0, **kwa): """ Command line version handler """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/passcode/remove.py b/src/keri/app/cli/commands/passcode/remove.py index 68c0aabe2..8feba43d2 100644 --- a/src/keri/app/cli/commands/passcode/remove.py +++ b/src/keri/app/cli/commands/passcode/remove.py @@ -19,7 +19,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='existing 22 character encryption passcode for keystore', +parser.add_argument('--passcode', '-p', help='existing 21 character encryption passcode for keystore', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/passcode/set.py b/src/keri/app/cli/commands/passcode/set.py index 3081a780b..99e7b9b51 100644 --- a/src/keri/app/cli/commands/passcode/set.py +++ b/src/keri/app/cli/commands/passcode/set.py @@ -7,22 +7,28 @@ import getpass from keri import help + from hio.base import doing -from keri.app.cli.common import existing -from keri.core import coring from keri.kering import ConfigurationError -logger = help.ogler.getLogger() +from keri import core +from keri.core import coring + +from keri.app.cli.common import existing + + + +logger = help.ogler.getLogger() # I think this should be keri.help not hio.help parser = argparse.ArgumentParser(description='Initialize a prefix') parser.set_defaults(handler=lambda args: set_passcode(args), transferable=True) -parser.add_argument("--new", help="new 22 character encryption passcode for keystore", required=False, default=None) +parser.add_argument("--new", help="new 21 character encryption passcode for keystore", required=False, default=None) parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='existing 22 character encryption passcode for keystore', +parser.add_argument('--passcode', '-p', help='existing 21 character encryption passcode for keystore', dest="bran", default=None) # passcode => bran @@ -50,7 +56,7 @@ def set_passcode(tymth, tock=0.0, **opts): with existing.existingHby(name=name, base=base, bran=bran) as hby: if newpasscode is None: - print("Re-encrypting keystore, please enter the new 22 character passcode:") + print("Re-encrypting keystore, please enter the new 21 character passcode:") while True: newpasscode = getpass.getpass("New passcode: ") retry = getpass.getpass("Re-enter new passcode: ") @@ -61,7 +67,7 @@ def set_passcode(tymth, tock=0.0, **opts): break bran = coring.MtrDex.Salt_128 + newpasscode[:22] # qb64 salt for seed - signer = coring.Salter(qb64=bran).signer(transferable=False, + signer = core.Salter(qb64=bran).signer(transferable=False, temp=False) seed = signer.qb64 aeid = signer.verfer.qb64 diff --git a/src/keri/app/cli/commands/query.py b/src/keri/app/cli/commands/query.py index 53e6db690..3505e9411 100644 --- a/src/keri/app/cli/commands/query.py +++ b/src/keri/app/cli/commands/query.py @@ -28,7 +28,7 @@ parser.add_argument('--prefix', help='QB64 identifier to query', default="", required=True) # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) diff --git a/src/keri/app/cli/commands/rename.py b/src/keri/app/cli/commands/rename.py index 7d975d05c..83251b88f 100644 --- a/src/keri/app/cli/commands/rename.py +++ b/src/keri/app/cli/commands/rename.py @@ -22,7 +22,7 @@ required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the identifier prefix', required=True) parser.add_argument('new', help='new human readable alias for the identifier') -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran @@ -45,12 +45,22 @@ def rename(tymth, tock=0.0, **opts): try: with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (hby, hab): - habord = hab.db.habs.get(keys=alias) - hab.db.habs.put(keys=newAlias, - val=habord) - hab.db.habs.rem(keys=alias) + if hby.habByName(newAlias) is not None: + print(f"{newAlias} is already in use") + + if (pre := hab.db.names.get(keys=("", alias))) is not None: + + habord = hab.db.habs.get(keys=pre) + habord.name = newAlias + hab.db.habs.pin(keys=habord.hid, + val=habord) + hab.db.names.pin(keys=("", newAlias), val=pre) + hab.db.names.rem(keys=("", alias)) + + print(f"Hab {alias} renamed to {newAlias}") + else: + raise ConfigurationError(f"No AID with name {alias} found") - print(f"Hab {alias} renamed to {newAlias}") except ConfigurationError as e: print(f"identifier prefix for {name} does not exist, incept must be run first", ) diff --git a/src/keri/app/cli/commands/rollback.py b/src/keri/app/cli/commands/rollback.py index 1cfce3b7c..249590bad 100644 --- a/src/keri/app/cli/commands/rollback.py +++ b/src/keri/app/cli/commands/rollback.py @@ -25,7 +25,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/rotate.py b/src/keri/app/cli/commands/rotate.py index 8f0152c60..bd1868683 100644 --- a/src/keri/app/cli/commands/rotate.py +++ b/src/keri/app/cli/commands/rotate.py @@ -11,6 +11,7 @@ from keri import kering from keri.app.cli.common import rotating, existing, config from keri.core import coring +from keri.help import helping from ... import habbing, agenting, indirecting, delegating, forwarding parser = argparse.ArgumentParser(description='Rotate keys') @@ -19,14 +20,20 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--file', '-f', help='file path of config options (JSON) for rotation', default="", required=False) parser.add_argument('--next-count', '-C', help='Count of pre-rotated keys (signing keys after next rotation).', default=None, type=int, required=False) parser.add_argument("--receipt-endpoint", help="Attempt to connect to witness receipt endpoint for witness receipts.", dest="endpoint", action='store_true') -parser.add_argument("--proxy", help="alias for delegation communication proxy", default="") +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') +parser.add_argument('--code', help=': formatted witness auth codes. Can appear multiple times', + default=[], action="append", required=False) +parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False) + +parser.add_argument("--proxy", help="alias for delegation communication proxy", default=None) rotating.addRotationArgs(parser) @@ -60,7 +67,8 @@ def rotate(args): cuts=opts.witsCut, adds=opts.witsAdd, isith=opts.isith, nsith=opts.nsith, count=opts.ncount, toad=opts.toad, - data=opts.data, proxy=args.proxy) + data=opts.data, proxy=args.proxy, authenticate=args.authenticate, + codes=args.code, codeTime=args.code_time) doers = [rotDoer] @@ -115,7 +123,8 @@ class RotateDoer(doing.DoDoer): """ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=None, count=None, - toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None): + toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None, authenticate=False, + codes=None, codeTime=None): """ Returns DoDoer with all registered Doers needed to perform rotation. @@ -139,7 +148,9 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.toad = toad self.data = data self.endpoint = endpoint - self.proxy = proxy + self.authenticate = authenticate + self.codes = codes if codes is not None else [] + self.codeTime = codeTime self.wits = wits if wits is not None else [] self.cuts = cuts if cuts is not None else [] @@ -147,14 +158,16 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No self.hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=self.hby) # setup doer - self.swain = delegating.Sealer(hby=self.hby) + + self.proxy = self.hby.habByName(proxy) if proxy is not None else None + self.swain = delegating.Anchorer(hby=self.hby, proxy=self.proxy) self.postman = forwarding.Poster(hby=self.hby) self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/receipt', "/replay", "/reply"]) doers = [self.hbyDoer, self.mbx, self.swain, self.postman, doing.doify(self.rotateDo)] super(RotateDoer, self).__init__(doers=doers) - def rotateDo(self, tymth, tock=0.0): + def rotateDo(self, tymth, tock=0.0, **kwa): """ Returns: doifiable Doist compatible generator method Usage: @@ -187,21 +200,34 @@ def rotateDo(self, tymth, tock=0.0): cuts=list(self.cuts), adds=list(self.adds), data=self.data) - if hab.kever.delegator: - self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, proxy=self.hby.habByName(self.proxy)) + auths = {} + if self.authenticate: + codeTime = helping.fromIso8601(self.codeTime) if self.codeTime is not None else helping.nowIso8601() + for arg in self.codes: + (wit, code) = arg.split(":") + auths[wit] = f"{code}#{codeTime}" + + for wit in hab.kever.wits: + if wit in auths: + continue + code = input(f"Enter code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" + + if hab.kever.delpre: + self.swain.delegation(pre=hab.pre, sn=hab.kever.sn, auths=auths, proxy=self.proxy) print("Waiting for delegation approval...") while not self.swain.complete(hab.kever.prefixer, coring.Seqner(sn=hab.kever.sn)): yield self.tock elif hab.kever.wits: if self.endpoint: - yield from receiptor.receipt(hab.pre, sn=hab.kever.sn) + yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) else: for wit in self.adds: self.mbx.addPoller(hab, witness=wit) print("Waiting for witness receipts...") - witDoer = agenting.WitnessReceiptor(hby=self.hby) + witDoer = agenting.WitnessReceiptor(hby=self.hby, auths=auths) self.extend(doers=[witDoer]) yield self.tock @@ -211,8 +237,12 @@ def rotateDo(self, tymth, tock=0.0): self.remove([witDoer]) - if hab.kever.delegator: - yield from self.postman.sendEvent(hab=hab, fn=hab.kever.sn) + if hab.kever.delpre: + if self.proxy is not None: + sender = self.proxy + else: + sender = hab + yield from self.postman.sendEventToDelegator(hab=hab, sender=sender, fn=hab.kever.sn) print(f'Prefix {hab.pre}') print(f'New Sequence No. {hab.kever.sn}') diff --git a/src/keri/app/cli/commands/salt.py b/src/keri/app/cli/commands/salt.py index 274ccd5de..e70ee36a4 100644 --- a/src/keri/app/cli/commands/salt.py +++ b/src/keri/app/cli/commands/salt.py @@ -8,7 +8,7 @@ import pysodium from hio.base import doing -from keri.core import coring +from keri import core parser = argparse.ArgumentParser(description='Print a new random passcode') parser.set_defaults(handler=lambda args: handler(args)) @@ -18,9 +18,9 @@ def handler(_): return [doing.doify(passcode)] -def passcode(tymth, tock=0.0): +def passcode(tymth, tock=0.0, **kwa): """ Command line version handler """ _ = (yield tock) - print(coring.Salter(raw=pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES)).qb64) + print(core.Salter(raw=pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES)).qb64) diff --git a/src/keri/app/cli/commands/sign.py b/src/keri/app/cli/commands/sign.py index fa2c44d8b..b6324d6e4 100644 --- a/src/keri/app/cli/commands/sign.py +++ b/src/keri/app/cli/commands/sign.py @@ -18,7 +18,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--text', '-t', help='Text or file (starts with "@") to sign', required=True) diff --git a/src/keri/app/cli/commands/ssh/export.py b/src/keri/app/cli/commands/ssh/export.py index 0b45ad472..21d1e15cb 100644 --- a/src/keri/app/cli/commands/ssh/export.py +++ b/src/keri/app/cli/commands/ssh/export.py @@ -26,7 +26,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--private", help="export private key instead of public key", action="store_true") diff --git a/src/keri/app/cli/commands/status.py b/src/keri/app/cli/commands/status.py index 130bd50ee..741c8acfb 100644 --- a/src/keri/app/cli/commands/status.py +++ b/src/keri/app/cli/commands/status.py @@ -22,7 +22,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/time.py b/src/keri/app/cli/commands/time.py index fa48a1638..b17f317c9 100644 --- a/src/keri/app/cli/commands/time.py +++ b/src/keri/app/cli/commands/time.py @@ -17,7 +17,7 @@ def handler(_): return [doing.doify(time)] -def time(tymth, tock=0.0): +def time(tymth, tock=0.0, **kwa): """ time """ _ = (yield tock) diff --git a/src/keri/app/cli/commands/vc/create.py b/src/keri/app/cli/commands/vc/create.py index ce8c5bf50..282bd9de9 100644 --- a/src/keri/app/cli/commands/vc/create.py +++ b/src/keri/app/cli/commands/vc/create.py @@ -6,9 +6,11 @@ from hio.base import doing from keri import kering +from keri import core +from keri.core import coring, eventing, serdering + from keri.app import indirecting, habbing, grouping, connecting, forwarding, signing, notifying from keri.app.cli.common import existing -from keri.core import coring, eventing, serdering from keri.help import helping from keri.peer import exchanging from keri.vdr import credentialing, verifying @@ -37,9 +39,11 @@ parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) parser.add_argument("--private", help="flag to indicate if this credential needs privacy preserving features", action="store_true") -parser.add_argument("--private-credential-nonce", help="(str) nonce for vc", action="store") -parser.add_argument("--private-subject-nonce", help="(str) nonce for subject", action="store") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument("--private-credential-nonce", help="nonce for vc", + action="store") +parser.add_argument("--private-subject-nonce", help="nonce for subject", + action="store") +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--time", help="timestamp for the credential creation", required=False, default=None) @@ -199,7 +203,7 @@ def __init__(self, name, alias, base, bran, registryName=None, schema=None, edge doers.extend([doing.doify(self.createDo)]) super(CredentialIssuer, self).__init__(doers=doers) - def createDo(self, tymth, tock=0.0): + def createDo(self, tymth, tock=0.0, **kwa): """ Issue Credential doer method @@ -218,9 +222,9 @@ def createDo(self, tymth, tock=0.0): dt = self.creder.attrib["dt"] if "dt" in self.creder.attrib else helping.nowIso8601() iserder = registry.issue(said=self.creder.said, dt=dt) - vcid = iserder.ked["i"] - rseq = coring.Seqner(snh=iserder.ked["s"]) - rseal = eventing.SealEvent(vcid, rseq.snh, iserder.said) + #vcid = iserder.ked["i"] + #rseq = coring.Seqner(snh=iserder.ked["s"]) + rseal = eventing.SealEvent(iserder.pre, iserder.snh, iserder.said) rseal = dict(i=rseal.i, s=rseal.s, d=rseal.d) if registry.estOnly: @@ -229,11 +233,12 @@ def createDo(self, tymth, tock=0.0): else: anc = hab.interact(data=[rseal]) - aserder = serdering.SerderKERI(raw=anc) # coring.Serder(raw=anc) + aserder = serdering.SerderKERI(raw=anc) self.credentialer.issue(self.creder, iserder) self.registrar.issue(self.creder, iserder, aserder) - acdc = signing.serialize(self.creder, coring.Prefixer(qb64=iserder.pre), coring.Seqner(sn=iserder.sn), + acdc = signing.serialize(self.creder, coring.Prefixer(qb64=iserder.pre), + core.Number(num=iserder.sn, code=core.NumDex.Huge), coring.Saider(qb64=iserder.said)) if isinstance(self.hab, habbing.GroupHab): diff --git a/src/keri/app/cli/commands/vc/export.py b/src/keri/app/cli/commands/vc/export.py index d4821dd7b..66b26a52b 100644 --- a/src/keri/app/cli/commands/vc/export.py +++ b/src/keri/app/cli/commands/vc/export.py @@ -25,7 +25,7 @@ required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--said", "-s", help="SAID of the credential to export.", required=False, default=None) @@ -78,7 +78,7 @@ def __init__(self, name, alias, base, bran, said, tels, kels, chains, files): super(ExportDoer, self).__init__(doers=doers) - def exportDo(self, tymth, tock=0.0): + def exportDo(self, tymth, tock=0.0, **kwa): """ Export credential from store and any related material Parameters: diff --git a/src/keri/app/cli/commands/vc/list.py b/src/keri/app/cli/commands/vc/list.py index d1528b31f..1d46f81b2 100644 --- a/src/keri/app/cli/commands/vc/list.py +++ b/src/keri/app/cli/commands/vc/list.py @@ -29,7 +29,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print full JSON of all credentials", action="store_true") @@ -80,7 +80,7 @@ def __init__(self, name, alias, base, bran, verbose=False, poll=False, said=Fals super(ListDoer, self).__init__(doers=doers) - def listDo(self, tymth, tock=0.0): + def listDo(self, tymth, tock=0.0, **kwa): """ Check for any credential messages in mailboxes and list all held credentials Parameters: diff --git a/src/keri/app/cli/commands/vc/registry/incept.py b/src/keri/app/cli/commands/vc/registry/incept.py index 6b6b3243d..4de3d4a81 100644 --- a/src/keri/app/cli/commands/vc/registry/incept.py +++ b/src/keri/app/cli/commands/vc/registry/incept.py @@ -7,7 +7,7 @@ from keri.app.cli.common import existing from keri.app.habbing import GroupHab from keri.app.notifying import Notifier -from keri.core import serdering +from keri.core import coring, serdering from keri.core.eventing import SealEvent from keri.peer import exchanging from keri.vdr import credentialing @@ -22,7 +22,7 @@ default=None) parser.add_argument('--nonce', help='Unique random value to seed the credential registry', default=None, required=False) -parser.add_argument("--no-backers", "-nb", help="do not allow setting up backers different from the ahcnoring KEL " +parser.add_argument("--no-backers", "-nb", help="do not allow setting up backers different from the anchoring KEL " "witnesses", default=True, action="store") parser.add_argument('--backers', help='New set of backers different from the anchoring KEL witnesses. Can ' 'appear multiple times', metavar="", default=[], action="append", @@ -32,7 +32,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--usage', '-u', help='For multisig issuers, a message to other participants about how this' ' registry is to be used', @@ -74,7 +74,7 @@ def __init__(self, name, base, alias, bran, registryName, usage, **kwa): name (str): name of habery and shared db and file path base (str): optional if "" path component of shared db and files. alias(str): human readable name of identifier to use for registry inception - bran (str): Base64 22 char string that is used as base material for + bran (str): Base64 21 char string that is used as base material for seed. bran allows alphanumeric passcodes generated by key managers like 1password to be key store for seed. registryName(str): human readable name for newly created registry @@ -132,7 +132,7 @@ def inceptDo(self, tymth, tock=0.0, **kwa): else: anc = hab.interact(data=[rseal]) - aserder = serdering.SerderKERI(raw=bytes(anc)) # coring.Serder(raw=bytes(anc)) + aserder = serdering.SerderKERI(raw=bytes(anc)) self.registrar.incept(iserder=registry.vcp, anc=aserder) if isinstance(hab, GroupHab): diff --git a/src/keri/app/cli/commands/vc/registry/list.py b/src/keri/app/cli/commands/vc/registry/list.py index 90d4c923e..c7d99380e 100644 --- a/src/keri/app/cli/commands/vc/registry/list.py +++ b/src/keri/app/cli/commands/vc/registry/list.py @@ -21,7 +21,7 @@ parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") diff --git a/src/keri/app/cli/commands/vc/registry/status.py b/src/keri/app/cli/commands/vc/registry/status.py index 55de01ca2..a6c5e5dea 100644 --- a/src/keri/app/cli/commands/vc/registry/status.py +++ b/src/keri/app/cli/commands/vc/registry/status.py @@ -5,7 +5,7 @@ from keri.app import indirecting, habbing, grouping from keri.app.cli.common import existing -from keri.core import serdering +from keri.core import coring, serdering from keri.vdr import credentialing logger = help.ogler.getLogger() @@ -18,7 +18,7 @@ default=None, required=True) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--verbose", "-V", help="print JSON of all current events", action="store_true") @@ -61,7 +61,7 @@ def __init__(self, name, base, bran, registryName, verbose): doers.extend([doing.doify(self.statusDo)]) super(RegistryStatusor, self).__init__(doers=doers) - def statusDo(self, tymth, tock=0.0): + def statusDo(self, tymth, tock=0.0, **kwa): """ Process incoming messages to incept a credential registry Parameters: diff --git a/src/keri/app/cli/commands/vc/revoke.py b/src/keri/app/cli/commands/vc/revoke.py index a37059255..a04a1220b 100644 --- a/src/keri/app/cli/commands/vc/revoke.py +++ b/src/keri/app/cli/commands/vc/revoke.py @@ -24,7 +24,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--said', help='is SAID vc content qb64') parser.add_argument('--send', help='alias of contact to send the revocation events to (can be repeated)', @@ -71,7 +71,7 @@ def __init__(self, name, alias, said, base, bran, registryName, send, timestamp, doers.extend([doing.doify(self.revokeDo)]) super(RevokeDoer, self).__init__(doers=doers, **kwa) - def revokeDo(self, tymth, tock=0.0): + def revokeDo(self, tymth, tock=0.0, **kwa): """ Revoke Credential doer method diff --git a/src/keri/app/cli/commands/verify.py b/src/keri/app/cli/commands/verify.py index aa75a1c56..17033bb20 100644 --- a/src/keri/app/cli/commands/verify.py +++ b/src/keri/app/cli/commands/verify.py @@ -9,7 +9,7 @@ from keri import kering from keri.app.cli.common import existing -from keri.core import coring +from keri.core import coring, indexing parser = argparse.ArgumentParser(description='Verify signature(s) on arbitrary data') parser.set_defaults(handler=lambda args: handler(args)) @@ -17,7 +17,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--prefix", help="Identifier prefix of the signer", required=True) @@ -49,7 +49,7 @@ def verify(tymth, tock=0.0, **opts): base = args.base bran = args.bran - sigers = [coring.Siger(qb64=sig) for sig in args.signature] + sigers = [indexing.Siger(qb64=sig) for sig in args.signature] try: with existing.existingHab(name=name, alias=alias, base=base, bran=bran) as (_, hab): diff --git a/src/keri/app/cli/commands/version.py b/src/keri/app/cli/commands/version.py index 98cc81c5a..d2a9a0385 100644 --- a/src/keri/app/cli/commands/version.py +++ b/src/keri/app/cli/commands/version.py @@ -16,7 +16,7 @@ default=None) parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran diff --git a/src/keri/app/cli/commands/watcher/__init__.py b/src/keri/app/cli/commands/watcher/__init__.py index e69de29bb..dbf28aaaf 100644 --- a/src/keri/app/cli/commands/watcher/__init__.py +++ b/src/keri/app/cli/commands/watcher/__init__.py @@ -0,0 +1,6 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.app.cli.commands.watcher Package +""" + diff --git a/src/keri/app/cli/commands/watcher/add.py b/src/keri/app/cli/commands/watcher/add.py new file mode 100644 index 000000000..a28454d5e --- /dev/null +++ b/src/keri/app/cli/commands/watcher/add.py @@ -0,0 +1,146 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app import connecting, habbing, forwarding +from keri.app.cli.common import existing +from keri.core import serdering +from keri.kering import Roles + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Add AID or Alias to list of AIDs for a watcher to watch') +parser.set_defaults(handler=lambda args: add(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--watcher", '-w', help="the watcher AID or alias to add to", required=True) +parser.add_argument("--watched", '-W', help="the watched AID or alias to add") + + +def add(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + ed = AddDoer(name=args.name, + alias=args.alias, + base=args.base, + bran=args.bran, + watcher=args.watcher, + watched=args.watched) + return [ed] + + +class AddDoer(doing.DoDoer): + + def __init__(self, name, alias, base, bran, watcher, watched): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.org = connecting.Organizer(hby=self.hby) + + wat = None + if watcher in self.hby.kevers: + wat = watcher + else: + contacts = self.org.find("alias", watcher) + for contact in contacts: + if contact['alias'] == watcher: + wat = contact['id'] + + if not wat: + raise ValueError(f"unknown watcher {watcher}") + + watd = None + if watched in self.hby.kevers: + watd = watched + else: + contacts = self.org.find("alias", watched) + for contact in contacts: + if contact['alias'] == watched: + watd = contact['id'] + + if not watd: + raise ValueError(f"unknown watched {watched}") + + self.watcher = wat + self.watched = watd + + self.oobi = None + for (key,), obr in self.hby.db.roobi.getItemIter(): + if obr.cid == watd: + self.oobi = key + + if not self.oobi: + raise ValueError(f"no valid oobi for watched {self.watched}") + + doers = [doing.doify(self.addDo)] + + super(AddDoer, self).__init__(doers=doers) + + def addDo(self, tymth, tock=0.0, **kwa): + """ Add an AID to a watcher's list of AIDs to watch + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + if isinstance(self.hab, habbing.GroupHab): + raise ValueError("watchers for multisig AIDs not currently supported") + + ender = self.hab.db.ends.get(keys=(self.hab.pre, Roles.watcher, self.watcher)) + if not ender or not ender.allowed: + msg = self.hab.reply(route="/end/role/add", + data=dict(cid=self.hab.pre, role=Roles.watcher, eid=self.watcher)) + self.hab.psr.parseOne(ims=msg) + + postman = forwarding.StreamPoster(hby=self.hby, hab=self.hab, recp=self.watcher, topic="reply") + for msg in self.hab.db.cloneDelegation(self.hab.kever): + serder = serdering.SerderKERI(raw=msg) + postman.send(serder=serder, attachment=msg[serder.size:]) + + for msg in self.hab.db.clonePreIter(pre=self.hab.pre): + serder = serdering.SerderKERI(raw=msg) + postman.send(serder=serder, attachment=msg[serder.size:]) + + data = dict(cid=self.hab.pre, + oid=self.watched, + oobi=self.oobi) + + route = f"/watcher/{self.watcher}/add" + msg = self.hab.reply(route=route, data=data) + self.hab.psr.parseOne(ims=bytes(msg)) + rpy = serdering.SerderKERI(raw=msg) + postman.send(serder=rpy, attachment=msg[rpy.size:]) + + doer = doing.DoDoer(doers=postman.deliver()) + self.extend([doer]) + + while not doer.done: + yield self.tock + + print(f"Request to add {self.watched} to watcher {self.watcher} submitted.") diff --git a/src/keri/app/cli/commands/watcher/adjudicate.py b/src/keri/app/cli/commands/watcher/adjudicate.py new file mode 100644 index 000000000..0f441e4b1 --- /dev/null +++ b/src/keri/app/cli/commands/watcher/adjudicate.py @@ -0,0 +1,160 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import datetime +import random +import sys + +from keri import help +from hio.base import doing + +from keri.app import connecting, indirecting, querying, watching +from keri.app.cli.common import existing +from keri.app.watching import diffState, States +from keri.help import helping +from keri.kering import ConfigurationError +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Perform key event adjudication on any new key state from watchers.') +parser.set_defaults(handler=lambda args: handle(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--toad', '-t', default=None, required=False, type=int, + help='int of watcher threshold (threshold of acceptable duplicity)', ) +parser.add_argument("--watched", '-W', help="the watched AID or alias to add", required=True) +parser.add_argument("--poll", "-P", help="Poll mailboxes for any issued credentials", action="store_true") + + +def handle(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ + + kwa = dict(args=args) + adjudicator = AdjudicationDoer(**kwa) + + return [adjudicator] + + +class AdjudicationDoer(doing.DoDoer): + + def __init__(self, **kwa): + args = kwa["args"] + base = args.base + bran = args.bran + self.name = args.name + self.alias = args.alias + self.watched = args.watched + self.poll = args.poll + self.toad = args.toad + + self.hby = existing.setupHby(name=self.name, base=base, bran=bran) + self.mbx = indirecting.MailboxDirector(hby=self.hby, topics=['/reply', '/replay']) + doers = [doing.doify(self.adjudicate, **kwa), self.mbx] + + super(AdjudicationDoer, self).__init__(**kwa, doers=doers) + + def adjudicate(self, tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + + try: + org = connecting.Organizer(hby=self.hby) + + if self.poll: + end = helping.nowUTC() + datetime.timedelta(seconds=5) + sys.stdout.write(f"Polling mailboxes") + sys.stdout.flush() + while helping.nowUTC() < end: + sys.stdout.write(".") + sys.stdout.flush() + yield 1.0 + print("\n") + + if self.watched in self.hby.kevers: + watd = self.watched + else: + watd = org.find("alias", self.watched) + if len(watd) != 1: + raise ValueError(f"invalid recipient {self.watched}") + watd = watd[0]['id'] + + if not watd: + raise ValueError(f"unknown watched {self.watched}") + + if self.alias is None: + self.alias = existing.aliasInput(self.hby) + + hab = self.hby.habByName(self.alias) + if hab is None: + raise ValueError(f"unknown alias {self.alias}") + + adj = watching.Adjudicator(hby=self.hby, hab=hab) + adjDoer = watching.AdjudicationDoer(adj) + self.extend([adjDoer]) + + adj.msgs.append(dict(oid=self.watched, toad=self.toad)) + + while not adj.cues: + yield self.tock + + cue = adj.cues.pull() + kin = cue['kin'] + + match kin: + case "keyStateConsistent": + states = cue['states'] + wids = cue["wids"] + print(f"Local key state is consistent with the {len(states)} (out of " + f"{len(wids)} total) watchers that responded") + + case "keyStateLagging": + bhds = cue["behinds"] + print("The following watchers are behind the local KEL:") + for state in bhds: + print(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + print(f"Recommend the checking those watchers for access to {self.watched} witnesses") + + case "keyStateUpdate": + ahds = cue["aheads"] + print(f"Threshold ({self.toad}) satisfying number of watchers ({len(ahds)}) are ahead") + for state in ahds: + print(f"\tWatcher {state.wit} at Seq No. {state.sn} with digest: {state.dig}") + + print("Submitting query to update local copy of latest events.") + state = random.choice(ahds) + querier = querying.SeqNoQuerier(hby=self.hby, hab=hab, pre=self.watched, sn=state.sn, + wits=[state.wit]) + self.extend([querier]) + + while not querier.done: + yield self.tock + + case "keyStateDuplicitous": + dups = cue["dups"] + print(f"Duplicity detected for AID {self.watched}, local key state remains intact.") + for state in dups: + print(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + self.remove([self.mbx, adjDoer]) + + except ConfigurationError as e: + print(f"identifier prefix for {self.name} does not exist, incept must be run first", ) + return -1 + diff --git a/src/keri/app/cli/commands/watcher/list.py b/src/keri/app/cli/commands/watcher/list.py index 43d50ac18..47644e599 100644 --- a/src/keri/app/cli/commands/watcher/list.py +++ b/src/keri/app/cli/commands/watcher/list.py @@ -1,16 +1,40 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" import argparse +from keri import help from hio.base import doing -from keri import kering +from keri.app import connecting from keri.app.cli.common import existing +from keri.kering import ConfigurationError, Roles + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='List current watchers') +parser.set_defaults(handler=lambda args: handle(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran -parser = argparse.ArgumentParser(description='List set of watchers for this environment (Habitat)') -parser.set_defaults(handler=lambda args: handler(args)) -parser.add_argument('--name', '-n', help='Human readable reference', required=True) +def handle(args): + """ Command line handler for adding an aid to a watcher's list of AIds to watch + + Parameters: + args(Namespace): parsed command line arguments + + """ -def handler(args): kwa = dict(args=args) return [doing.doify(listWatchers, **kwa)] @@ -23,15 +47,22 @@ def listWatchers(tymth, tock=0.0, **opts): args = opts["args"] name = args.name alias = args.alias + base = args.base + bran = args.bran try: - with existing.existingHab(name=name, alias=alias) as hab: - habr = hab.db.habs.get(name) - print("Watcher Set:") - for wat in habr.watchers: - print("\t{}".format(wat)) + with existing.existingHby(name=name, base=base, bran=bran) as hby: + org = connecting.Organizer(hby=hby) + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) - except kering.ConfigurationError: + for (aid, role, eid), ender in hab.db.ends.getItemIter(keys=(hab.pre, Roles.watcher, )): + if ender.allowed: + contact = org.get(eid) + print(f"{contact['alias']}: {eid}") + + except ConfigurationError as e: print(f"identifier prefix for {name} does not exist, incept must be run first", ) return -1 - diff --git a/src/keri/app/cli/commands/watcher/start.py b/src/keri/app/cli/commands/watcher/start.py deleted file mode 100644 index 5c9801daf..000000000 --- a/src/keri/app/cli/commands/watcher/start.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- encoding: utf-8 -*- -""" -keri.kli.commands.watcher module - -""" -import argparse -import logging - -import falcon -from hio.core import http -from hio.core.tcp import serving - -from keri import help -from keri.app import directing, indirecting, habbing, storing - -parser = argparse.ArgumentParser(description='Start watcher') -parser.set_defaults(handler=lambda args: startWatcher(args)) -parser.add_argument('-H', '--http', - action='store', - default=5651, - help="Local port number the HTTP server listens on. Default is 5651.") -parser.add_argument('-T', '--tcp', - action='store', - default=5652, - help="Local port number the HTTP server listens on. Default is 5652.") -parser.add_argument('-n', '--name', - action='store', - default="watcher", - help="Name of controller. Default is watcher.") -parser.add_argument('--controller', - action='store', - default="", - help="Identifier prefix of controller of this watcher") -parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', - required=False, default="") -parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', - dest="bran", default=None) # passcode => bran - - -def startWatcher(args): - name = args.name - tcpPort = int(args.tcp) - httpPort = int(args.http) - - help.ogler.level = logging.INFO - help.ogler.reopen(name="keri", temp=True, clear=True) - doers = setupWatcher(name, controller=args.controller, alias=args.alias, base=args.base, bran=args.bran, - tcpPort=tcpPort, httpPort=httpPort) - return doers - - -def setupWatcher(name="watcher", controller=None, alias="watcher", base="", bran=None, tcpPort=5651, httpPort=5652): - """ - """ - - hby = habbing.Habery(name=name, base=base, bran=bran) - hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer - doers = [hbyDoer] - hab = hby.makeHab(name=alias, transferable=False) - - app = falcon.App(cors_enable=True) - - mbx = storing.Mailboxer(name=name) - rep = storing.Respondant(hby=hby, mbx=mbx) - - httpEnd = indirecting.HttpEnd(hab=hab, app=app, rep=rep, mbx=mbx) - app.add_route("/", httpEnd) - - server = http.Server(port=httpPort, app=app) - httpServerDoer = http.ServerDoer(server=server) - - server = serving.Server(host="", port=tcpPort) - serverDoer = serving.ServerDoer(server=server) - - directant = directing.Directant(hab=hab, server=server) - - doers.extend([directant, serverDoer, httpServerDoer, httpEnd, rep]) - - return doers diff --git a/src/keri/app/cli/commands/watcher/update.py b/src/keri/app/cli/commands/watcher/update.py deleted file mode 100644 index 3c37e5044..000000000 --- a/src/keri/app/cli/commands/watcher/update.py +++ /dev/null @@ -1,73 +0,0 @@ -import argparse - -from hio.base import doing - -from keri import kering -from keri.app import habbing -from keri.core import coring - -parser = argparse.ArgumentParser(description='Update set of watchers for this environment (Habitat)') -parser.set_defaults(handler=lambda args: handler(args)) -parser.add_argument('--name', '-n', help='Human readable reference', required=True) -parser.add_argument('--watchers', '-w', help='New set of watchers, replaces all existing watchers. Can appear ' - 'multiple times', metavar="", default=[], - action="append", required=False) -parser.add_argument('--cut', '-c', help='Watchers to remove. Can appear multiple times', metavar="", - default=[], - action="append", required=False) -parser.add_argument('--add', '-a', help='Watchers to add. Can appear multiple times', metavar="", - default=[], - action="append", required=False) - - -def handler(args): - kwa = dict(args=args) - return [doing.doify(updateWatchers, **kwa)] - - -def updateWatchers(tymth, tock=0.0, **opts): - """ Command line status handler - - """ - _ = (yield tock) - args = opts["args"] - name = args.name - - try: - with habbing.existingHabitat(name=name) as hab: - - habr = hab.db.habs.get(name) - - cuts = set(args.cut) - adds = set(args.add) - ewats = set(habr.watchers) - if args.watchers: - if args.add or args.cut: - raise kering.ConfigurationError("you can only specify witnesses or cuts and add") - - ewats = set(args.watchers) - else: - ewats |= adds - ewats -= cuts - - for wat in ewats: - pre = coring.Prefixer(qb64=wat) - if pre.transferable: - raise ValueError("{} is an illegal watcher identifier, must be non-transferable" - "".format(wat)) - - habr.watchers = list(ewats) - - hab.db.habs.pin(name, habr) - - print("New Watcher Set:") - for wat in habr.watchers: - print("\t{}".format(wat)) - - except kering.ConfigurationError: - print(f"identifier prefix for {name} does not exist, incept must be run first", ) - return -1 - except ValueError as e: - print(e.args[0]) - return -1 - diff --git a/src/keri/app/cli/commands/witness/authenticate.py b/src/keri/app/cli/commands/witness/authenticate.py new file mode 100644 index 000000000..f502739b2 --- /dev/null +++ b/src/keri/app/cli/commands/witness/authenticate.py @@ -0,0 +1,147 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse +import json +import sys + +import qrcode +from keri import help +from hio.base import doing +from hio.help import Hict + +from keri.app import httping, connecting +from keri.app.agenting import httpClient +from keri.app.cli.common import existing +from keri.app.httping import CESR_DESTINATION_HEADER +from keri.core import coring + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Perform authentication against an witness to get a OTP code') +parser.set_defaults(handler=lambda args: auth(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued', + required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument("--witness", '-w', help="the witness AID or alias to authenticate against", required=True) +parser.add_argument("--url-only", '-u', dest="url", help="display only the URL (no QR Code).", required=False, + action="store_true") + + +def auth(args): + """ Command line handler for authenticating against a witness by retrieving the secret or a TOTP + + Parameters: + args(Namespace): parsed command line arguments + + """ + + ed = AuthDoer(name=args.name, + alias=args.alias, + base=args.base, + bran=args.bran, + witness=args.witness, + urlOnly=args.url) + return [ed] + + +class AuthDoer(doing.DoDoer): + + def __init__(self, name, alias, base, bran, witness, urlOnly): + self.hby = existing.setupHby(name=name, base=base, bran=bran) + self.hab = self.hby.habByName(alias) + self.org = connecting.Organizer(hby=self.hby) + self.urlOnly = urlOnly + + if witness in self.hby.kevers: + wit = witness + else: + wit = self.org.find("alias", witness) + if len(wit) != 1: + raise ValueError(f"invalid recipient {witness}") + wit = wit[0]['id'] + + if not wit: + raise ValueError(f"unknown witness {witness}") + + self.witness = wit + self.clienter = httping.Clienter() + doers = [doing.doify(self.authDo), self.clienter] + + super(AuthDoer, self).__init__(doers=doers) + + def authDo(self, tymth, tock=0.0, **kwa): + """ Export credential from store and any related material + + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + + """ + # enter context + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + body = bytearray() + for msg in self.hab.db.clonePreIter(pre=self.hab.pre): + body.extend(msg) + + fargs = dict([("kel", body.decode("utf-8"))]) + + if self.hab.kever.delegated: + delkel = bytearray() + for msg in self.hab.db.clonePreIter(self.hab.kever.delpre): + delkel.extend(msg) + + fargs['delkel'] = delkel.decode("utf-8") + + headers = (Hict([ + ("Content-Type", "multipart/form-data"), + (CESR_DESTINATION_HEADER, self.witness) + ])) + + client, clientDoer = httpClient(self.hab, self.witness) + self.extend([clientDoer]) + + client.request( + method="POST", + path=f"{client.requester.path.rstrip('/')}/aids", + headers=headers, + fargs=fargs + ) + while not client.responses: + yield self.tock + + rep = client.respond() + if rep.status == 200: + data = json.loads(rep.body) + + totp = data["totp"] + m = coring.Matter(qb64=totp) # refactor this to use cipher + d = coring.Matter(qb64=self.hab.decrypt(ser=m.raw)) + otpurl = f"otpauth://totp/KERI:{self.witness}?secret={d.raw.decode('utf-8')}&issuer=KERI" + + if not self.urlOnly: + qr = qrcode.QRCode() + qr.add_data(otpurl) + + qr.print_ascii() + + print(otpurl) + + else: + raise ValueError(rep.body) + + self.remove([clientDoer, self.clienter]) diff --git a/src/keri/app/cli/commands/witness/catchup.py b/src/keri/app/cli/commands/witness/catchup.py new file mode 100644 index 000000000..8a563b9c3 --- /dev/null +++ b/src/keri/app/cli/commands/witness/catchup.py @@ -0,0 +1,169 @@ +# -*- encoding: utf-8 -*- +""" +keri.kli.commands module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app import habbing, agenting, httping +from keri.app.cli.common import existing +from keri.core import serdering + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='Send full KEL to a specific witness to catch it up') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the identifier prefix', required=True) + +# Authentication for keystore +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran +parser.add_argument('--witness', '-w', help='witness AID to send KEL to', required=True) +parser.add_argument('--force', '-f', help='send KEL even if witness is not in current witness pool', + action='store_true', default=False) +parser.add_argument('--verbose', '-V', help='show detailed progress for each event', + action='store_true', default=False) + + +def handler(args): + """ + Send full KEL to a specific witness to catch it up. + + Args: + args(Namespace): arguments object from command line + """ + + name = args.name + base = args.base + bran = args.bran + alias = args.alias + witness = args.witness + force = args.force + verbose = args.verbose + + doer = CatchupDoer(name=name, base=base, alias=alias, bran=bran, witness=witness, force=force, verbose=verbose) + + return [doer] + + +class CatchupDoer(doing.DoDoer): + """ DoDoer for sending full KEL to a witness to catch it up. + """ + + def __init__(self, name, base, alias, bran, witness, force=False, verbose=False): + + hby = existing.setupHby(name=name, base=base, bran=bran) + self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer + self.alias = alias + self.hby = hby + self.witness = witness + self.force = force + self.verbose = verbose + + doers = [self.hbyDoer, doing.doify(self.catchupDo)] + + super(CatchupDoer, self).__init__(doers=doers) + + def catchupDo(self, tymth, tock=0.0, **kwa): + """ + Parameters: + tymth (function): injected function wrapper closure returned by .tymen() of + Tymist instance. Calling tymth() returns associated Tymist .tyme. + tock (float): injected initial tock value + + Returns: doifiable Doist compatible generator method + """ + self.wind(tymth) + self.tock = tock + _ = (yield self.tock) + + hab = self.hby.habByName(name=self.alias) + if hab is None: + print(f"Error: unknown alias {self.alias}") + self.remove([self.hbyDoer]) + return + + # Validate witness is in the current witness list + if self.witness not in hab.kever.wits: + if not self.force: + print(f"Error: {self.witness} is not a witness for {self.alias}") + print(f"Current witnesses: {hab.kever.wits}") + print("Use --force to send KEL anyway") + self.remove([self.hbyDoer]) + return + else: + print(f"Warning: {self.witness} is not a witness for {self.alias}, forcing anyway") + + if self.verbose: + # Use verbose catchup with progress reporting + yield from self.verboseCatchup(hab) + else: + receiptor = agenting.Receiptor(hby=self.hby) + self.extend([receiptor]) + print(f"Sending full KEL to witness {self.witness}...") + yield from receiptor.catchup(hab.pre, self.witness) + self.remove([receiptor]) + + print(f"KEL sent successfully. Witness should now be at sn={hab.kever.sn}") + + self.remove([self.hbyDoer]) + + return + + def verboseCatchup(self, hab): + """Verbose catchup with progress reporting for each event.""" + from keri.app.agenting import httpClient + + client, clientDoer = httpClient(hab, self.witness) + self.extend([clientDoer]) + + print(f"Sending full KEL to witness {self.witness}...") + print(f"Controller KEL has {hab.kever.sn + 1} events (sn 0 to {hab.kever.sn})") + print() + + event_count = 0 + success_count = 0 + error_count = 0 + + for fmsg in hab.db.clonePreIter(pre=hab.pre): + # Parse to get event info + try: + serder = serdering.SerderKERI(raw=fmsg) + sn = serder.sn + ilk = serder.ilk + except Exception: + sn = '?' + ilk = '?' + + event_count += 1 + print(f" Sending event sn={sn} ilk={ilk}...", end=" ", flush=True) + + httping.streamCESRRequests(client=client, dest=self.witness, ims=bytearray(fmsg)) + while not client.responses: + yield self.tock + + # Check response + response = client.responses.popleft() + status = response.get('status', 'unknown') + + if status in (200, 204): + print(f"OK (HTTP {status})") + success_count += 1 + else: + print(f"ERROR (HTTP {status})") + error_count += 1 + body = response.get('body', b'') + if body: + print(f" Response: {body[:200]}") + + print() + print(f"Summary: {event_count} events sent, {success_count} succeeded, {error_count} errors") + + self.remove([clientDoer]) diff --git a/src/keri/app/cli/commands/witness/clean.py b/src/keri/app/cli/commands/witness/clean.py new file mode 100644 index 000000000..a56bead4b --- /dev/null +++ b/src/keri/app/cli/commands/witness/clean.py @@ -0,0 +1,45 @@ +# -*- encoding: utf-8 -*- +""" +keri.app.cli.commands.witness.clean module + +""" +import argparse + +from hio.base import doing + +from keri.app.cli.common import existing + +parser = argparse.ArgumentParser(description='Clean and re-verify witness database (offline)') +parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(clean, **kwa)] + + +def clean(tymth, tock=0.0, **opts): + """ Command line clean handler + + Cleans and re-verifies the database by creating a verified clone + and replacing the original. Must be run while witness is offline. + """ + _ = (yield tock) + + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + + print(f"Cleaning database for {name}...") + print("Database must be offline during this operation.") + + with existing.existingHby(name=name, base=base, bran=bran) as hby: + hby.db.clean() + + print(f"Database cleaned successfully.") diff --git a/src/keri/app/cli/commands/witness/demo.py b/src/keri/app/cli/commands/witness/demo.py index 8a9a27a31..3845bd060 100644 --- a/src/keri/app/cli/commands/witness/demo.py +++ b/src/keri/app/cli/commands/witness/demo.py @@ -12,10 +12,13 @@ from hio.base import doing -from keri.app import habbing, indirecting, configing -from keri.core.coring import Salter from keri import help +from keri.app import habbing, indirecting, configing + +from keri.core import Salter + + parser = argparse.ArgumentParser(description="Run a demo collection of witnesses") parser.add_argument("--loglevel", action="store", required=False, default=os.getenv("KERI_LOG_LEVEL", "CRITICAL"), help="Set log level to DEBUG | INFO | WARNING | ERROR | CRITICAL. Default is CRITICAL") @@ -30,8 +33,7 @@ def demo(args): Run set of three witnesses for demo """ - base_formatter = logging.Formatter( - '%(asctime)s [keri] %(module)s.%(funcName)s-%(lineno)s %(levelname)-8s %(message)s') + base_formatter = logging.Formatter('%(asctime)s [keri] %(module)s.%(funcName)s-%(lineno)s %(levelname)-8s %(message)s') base_formatter.default_msec_format = None help.ogler.baseConsoleHandler.setFormatter(base_formatter) help.ogler.level = logging.getLevelName(args.loglevel.upper()) @@ -66,10 +68,10 @@ def __init__(self, wan, wil, wes, wit, wub, wyz): self.wit = wit self.wub = wub self.wyz = wyz - + super(InitDoer, self).__init__(doers=[doing.doify(self.initialize)]) - def initialize(self, tymth, tock=0.0): + def initialize(self, tymth, tock=0.0, **kwa): # enter context self.wind(tymth) self.tock = tock diff --git a/src/keri/app/cli/commands/witness/inspect.py b/src/keri/app/cli/commands/witness/inspect.py new file mode 100644 index 000000000..098484299 --- /dev/null +++ b/src/keri/app/cli/commands/witness/inspect.py @@ -0,0 +1,351 @@ +# -*- encoding: utf-8 -*- +""" +keri.app.cli.commands.witness.inspect module + +Diagnostic command to inspect event databases and identify processing issues. +""" +import argparse +import json + +from hio.base import doing + +from keri.app.cli.common import existing +from keri.db import dbing +from keri.core import serdering + + +parser = argparse.ArgumentParser(description='Inspect event databases (.evts, .kels, .fels, .fons)') +parser.set_defaults(handler=lambda args: handler(args)) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) +parser.add_argument('--prefix', '-pre', help='filter by identifier prefix', default=None) +parser.add_argument('--verbose', '-v', help='show detailed event information', action='store_true', default=False) +parser.add_argument('--aid', '-a', help='show detailed kever info for specific AID', default=None) + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(inspect, **kwa)] + + +def inspect(tymth, tock=0.0, **opts): + """Command line inspect handler + + Inspects .evts, .kels, .fels, .fons databases and reports statistics + and potential issues that could cause event processing failures. + """ + _ = (yield tock) + + args = opts["args"] + name = args.name + base = args.base + bran = args.bran + prefix = args.prefix + verbose = args.verbose + aid = args.aid + + with existing.existingHby(name=name, base=base, bran=bran) as hby: + db = hby.db + + # If specific AID requested, show detailed kever info + if aid: + print(f"\n=== Detailed inspection for AID: {aid} ===\n") + + if aid in db.kevers: + kever = db.kevers[aid] + print(f"Kever loaded: YES") + print(f" Sequence Number (sn): {kever.sner.num}") + print(f" Last Event SAID: {kever.serder.said}") + print(f" Delegated: {kever.delegated}") + if kever.delegated: + print(f" Delegator: {kever.delegator}") + print(f" Witnesses: {kever.wits}") + print(f" Witness Threshold (toad): {kever.toader.num}") + + # Check witness receipts for current event + dgkey = dbing.dgKey(kever.serder.preb, kever.serder.saidb) + wigs = db.getWigs(dgkey) + print(f" Witness Receipts for current event: {len(wigs)}") + + # Check if fully witnessed + fully_witnessed = len(wigs) >= kever.toader.num + print(f" Fully Witnessed: {fully_witnessed}") + + print(f"\n--- Event-by-event witness receipts ---") + # Show witness receipts for each event in KEL + for sn, dig in db.getKelItemPreIter(aid.encode('utf-8') if isinstance(aid, str) else aid): + dgkey = dbing.dgKey(aid.encode('utf-8') if isinstance(aid, str) else aid, dig) + wigs = db.getWigs(dgkey) + sigs = db.getSigs(dgkey) + + # Get event type + try: + raw = db.getEvt(dgkey) + if raw: + serder = serdering.SerderKERI(raw=bytes(raw)) + ilk = serder.ilk + else: + ilk = "?" + except Exception: + ilk = "?" + + status = "✓" if len(wigs) >= kever.toader.num else "✗" + print(f" sn={sn:3d} ilk={ilk:3s} sigs={len(sigs) if sigs else 0} wigs={len(wigs) if wigs else 0}/{kever.toader.num} {status}") + + else: + print(f"Kever loaded: NO") + print(f" AID {aid} not found in kevers") + + # Check if there are any events for this AID + kel_count = 0 + max_sn = -1 + for sn, dig in db.getKelItemPreIter(aid.encode('utf-8') if isinstance(aid, str) else aid): + kel_count += 1 + max_sn = max(max_sn, sn) + + if kel_count > 0: + print(f" But found {kel_count} events in KEL (max sn={max_sn})") + print(f" This suggests events exist but kever failed to load") + else: + print(f" No events found in KEL for this AID") + + # Check escrows for this specific AID + print(f"\n--- Escrows for this AID ---") + aid_bytes = aid.encode('utf-8') if isinstance(aid, str) else aid + + pse_count = 0 + for key, val in db.getPseItemsNextIter(): + if aid_bytes in bytes(key): + pse_count += 1 + print(f" Partial Signed Escrow (PSE): {pse_count}") + + pwe_count = 0 + for key, val in db.getPweItemIter(): + if aid_bytes in bytes(key): + pwe_count += 1 + print(f" Partial Witnessed Escrow (PWE): {pwe_count}") + + ooe_count = 0 + for key, val in db.getOoeItemIter(): + if aid_bytes in bytes(key): + ooe_count += 1 + print(f" Out of Order Escrow (OOE): {ooe_count}") + + lde_count = 0 + for key, val in db.getLdeItemIter(): + if aid_bytes in bytes(key): + lde_count += 1 + print(f" Likely Duplicitous Escrow (LDE): {lde_count}") + + print() + return + + report = { + "summary": {}, + "prefixes": {}, + "issues": [] + } + + # Count total events in .evts + evts_count = 0 + evts_by_pre = {} + for key, val in db.getTopItemIter(db.evts): + evts_count += 1 + # Extract prefix from dgKey (pre.dig format) + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + pre = key_str.rsplit('.', 1)[0] + if prefix and pre != prefix: + continue + evts_by_pre[pre] = evts_by_pre.get(pre, 0) + 1 + + report["summary"]["evts_total"] = evts_count + + # Count entries in .kels (key event log) + kels_count = 0 + kels_by_pre = {} + for key, val in db.getTopItemIter(db.kels): + kels_count += 1 + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + pre = key_str.rsplit('.', 1)[0] + if prefix and pre != prefix: + continue + kels_by_pre[pre] = kels_by_pre.get(pre, 0) + 1 + + report["summary"]["kels_total"] = kels_count + + # Count entries in .fels (first seen event log) + fels_count = 0 + fels_by_pre = {} + for key, val in db.getTopItemIter(db.fels): + fels_count += 1 + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + pre = key_str.rsplit('.', 1)[0] + if prefix and pre != prefix: + continue + fels_by_pre[pre] = fels_by_pre.get(pre, 0) + 1 + + report["summary"]["fels_total"] = fels_count + + # Count entries in .fons (first seen ordinal number index) + fons_count = 0 + fons_by_pre = {} + for keys, val in db.fons.getItemIter(): + fons_count += 1 + if keys: + pre = keys[0].rsplit('.', 1)[0] if '.' in keys[0] else keys[0] + if prefix and pre != prefix: + continue + fons_by_pre[pre] = fons_by_pre.get(pre, 0) + 1 + + report["summary"]["fons_total"] = fons_count + + # Get all unique prefixes + all_prefixes = set(evts_by_pre.keys()) | set(kels_by_pre.keys()) | set(fels_by_pre.keys()) | set(fons_by_pre.keys()) + + if prefix: + all_prefixes = {prefix} if prefix in all_prefixes else set() + + # Analyze each prefix + for pre in sorted(all_prefixes): + pre_report = { + "evts": evts_by_pre.get(pre, 0), + "kels": kels_by_pre.get(pre, 0), + "fels": fels_by_pre.get(pre, 0), + "fons": fons_by_pre.get(pre, 0), + } + + # Check for issues + issues = [] + + # Events in .evts but not in .fels (not first-seen logged) + if pre_report["evts"] > 0 and pre_report["fels"] == 0: + issues.append("Events exist but none in first-seen log (.fels)") + + # Events in .evts but not in .kels (not in key event log) + if pre_report["evts"] > 0 and pre_report["kels"] == 0: + issues.append("Events exist but none in key event log (.kels)") + + # Mismatch between .fels and .fons + if pre_report["fels"] != pre_report["fons"]: + issues.append(f"Mismatch: .fels={pre_report['fels']} vs .fons={pre_report['fons']}") + + # More events than KEL entries could indicate duplicates or escrow + if pre_report["evts"] > pre_report["kels"] and pre_report["kels"] > 0: + diff = pre_report["evts"] - pre_report["kels"] + issues.append(f"More events ({pre_report['evts']}) than KEL entries ({pre_report['kels']}), {diff} may be escrowed/duplicitous") + + if issues: + pre_report["issues"] = issues + for issue in issues: + report["issues"].append(f"{pre[:16]}...: {issue}") + + report["prefixes"][pre] = pre_report + + # Verbose: show KEL sequence details + if verbose: + kel_details = [] + max_sn = -1 + sn_gaps = [] + prev_sn = -1 + + for sn, dig in db.getKelItemPreIter(pre.encode('utf-8') if isinstance(pre, str) else pre): + if prev_sn >= 0 and sn != prev_sn + 1: + sn_gaps.append((prev_sn, sn)) + prev_sn = sn + max_sn = max(max_sn, sn) + if len(kel_details) < 10: # Limit to first 10 + dig_str = bytes(dig).decode('utf-8') if isinstance(dig, (bytes, memoryview)) else str(dig) + kel_details.append({"sn": sn, "dig": dig_str[:24] + "..."}) + + pre_report["max_sn"] = max_sn + if sn_gaps: + pre_report["sn_gaps"] = sn_gaps + report["issues"].append(f"{pre[:16]}...: Sequence gaps found: {sn_gaps}") + if verbose and kel_details: + pre_report["kel_sample"] = kel_details + + # Check for FEL details + fel_details = [] + max_fn = -1 + for fn, dig in db.getFelItemPreIter(pre.encode('utf-8') if isinstance(pre, str) else pre): + max_fn = max(max_fn, fn) + if len(fel_details) < 10: + dig_str = bytes(dig).decode('utf-8') if isinstance(dig, (bytes, memoryview)) else str(dig) + fel_details.append({"fn": fn, "dig": dig_str[:24] + "..."}) + + pre_report["max_fn"] = max_fn + if verbose and fel_details: + pre_report["fel_sample"] = fel_details + + # Add kever information + report["summary"]["kevers_loaded"] = len(db.kevers) + report["summary"]["prefixes_count"] = len(all_prefixes) + + # Check escrows + escrows = { + "pse": 0, # Partial Signed Escrow + "pwe": 0, # Partial Witnessed Escrow + "ooe": 0, # Out of Order Escrow + "lde": 0, # Likely Duplicitous Escrow + } + + # Count PSE entries + for key, val in db.getPseItemsNextIter(): + escrows["pse"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"PSE escrow: {key_str[:40]}...") + + # Count PWE entries + for key, val in db.getPweItemIter(): + escrows["pwe"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"PWE escrow: {key_str[:40]}...") + + # Count OOE entries + for key, val in db.getOoeItemIter(): + escrows["ooe"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"OOE escrow: {key_str[:40]}...") + + # Count LDE entries + for key, val in db.getLdeItemIter(): + escrows["lde"] += 1 + if verbose: + key_str = bytes(key).decode('utf-8') if isinstance(key, (bytes, memoryview)) else str(key) + report["issues"].append(f"LDE escrow: {key_str[:40]}...") + + report["summary"]["escrows"] = escrows + if any(v > 0 for v in escrows.values()): + report["issues"].append(f"Escrows found: PSE={escrows['pse']}, PWE={escrows['pwe']}, OOE={escrows['ooe']}, LDE={escrows['lde']}") + + # Check for orphaned events (in .evts but not referenced) + if verbose: + # Get all digests from .kels and .fels + referenced_digs = set() + for key, val in db.getTopItemIter(db.kels): + referenced_digs.add(bytes(val).decode('utf-8')) + for key, val in db.getTopItemIter(db.fels): + referenced_digs.add(bytes(val).decode('utf-8')) + + orphaned = 0 + for key, val in db.getTopItemIter(db.evts): + key_str = bytes(key).decode('utf-8') + if '.' in key_str: + dig = key_str.rsplit('.', 1)[1] + if dig not in referenced_digs: + orphaned += 1 + + if orphaned > 0: + report["summary"]["orphaned_events"] = orphaned + report["issues"].append(f"Found {orphaned} orphaned events (in .evts but not referenced in .kels/.fels)") + + print(json.dumps(report, indent=2)) diff --git a/src/keri/app/cli/commands/witness/list.py b/src/keri/app/cli/commands/witness/list.py new file mode 100644 index 000000000..f86d8d9e5 --- /dev/null +++ b/src/keri/app/cli/commands/witness/list.py @@ -0,0 +1,56 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.kli.commands module + +""" +import argparse + +from keri import help +from hio.base import doing + +from keri.app.cli.common import displaying, existing +from keri.core import serdering +from keri.kering import ConfigurationError + +logger = help.ogler.getLogger() + +parser = argparse.ArgumentParser(description='List AIDs of witness for the provided AID') +parser.set_defaults(handler=lambda args: handler(args), + transferable=True) +parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True) +parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', + required=False, default="") +parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', default=None) +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', + dest="bran", default=None) # passcode => bran + + +def handler(args): + kwa = dict(args=args) + return [doing.doify(listWitnesses, **kwa)] + + +def listWitnesses(tymth, tock=0.0, **opts): + """ Command line status handler + + """ + _ = (yield tock) + args = opts["args"] + name = args.name + alias = args.alias + base = args.base + bran = args.bran + + try: + with existing.existingHby(name=name, base=base, bran=bran) as hby: + if alias is None: + alias = existing.aliasInput(hby) + + hab = hby.habByName(alias) + for idx, wit in enumerate(hab.kever.wits): + print(f'{wit}') + + except ConfigurationError as e: + print(f"identifier prefix for {name} does not exist, incept must be run first", ) + return -1 diff --git a/src/keri/app/cli/commands/witness/start.py b/src/keri/app/cli/commands/witness/start.py index 3ea02fe6e..c6acc7980 100644 --- a/src/keri/app/cli/commands/witness/start.py +++ b/src/keri/app/cli/commands/witness/start.py @@ -36,7 +36,7 @@ parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore', required=False, default="") parser.add_argument('--alias', '-a', help='human readable alias for the new identifier prefix', required=True) -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument("--config-dir", "-c", dest="configDir", help="directory override for configuration data") parser.add_argument('--config-file', @@ -55,7 +55,7 @@ def launch(args): help.ogler.level = logging.getLevelName(args.loglevel) - if(args.logfile != None): + if args.logfile is not None: help.ogler.headDirPath = args.logfile help.ogler.reopen(name=args.name, temp=False, clear=True) logger = help.ogler.getLogger() @@ -91,10 +91,13 @@ def runWitness(name="witness", base="", alias="witness", bran="", tcp=5631, http reopen=True) aeid = ks.gbls.get('aeid') + ks.close() cf = None if configFile is not None: cf = configing.Configer(name=configFile, headDirPath=configDir, temp=False, reopen=True, clear=False) + + aids = cf.get("aids", default=None) if cf is not None else None if aeid is None: hby = habbing.Habery(name=name, base=base, bran=bran, cf=cf) @@ -106,6 +109,7 @@ def runWitness(name="witness", base="", alias="witness", bran="", tcp=5631, http doers.extend(indirecting.setupWitness(alias=alias, hby=hby, + aids=aids, tcpPort=tcp, httpPort=http, keypath=keypath, diff --git a/src/keri/app/cli/commands/witness/submit.py b/src/keri/app/cli/commands/witness/submit.py index 652b0541e..37e9367b7 100644 --- a/src/keri/app/cli/commands/witness/submit.py +++ b/src/keri/app/cli/commands/witness/submit.py @@ -10,6 +10,7 @@ from keri.app import habbing, agenting, indirecting from keri.app.cli.common import existing, displaying +from keri.help import helping logger = help.ogler.getLogger() @@ -23,13 +24,17 @@ parser.add_argument("--config", "-c", help="directory override for configuration data") # Authentication for keystore -parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)', +parser.add_argument('--passcode', '-p', help='21 character encryption passcode for keystore (is not saved)', dest="bran", default=None) # passcode => bran parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for authentication ' 'and encryption of secrets in keystore', default=None) parser.add_argument('--force', action="store_true", required=False, help='True means to send witnesses all receipts even if we have a full compliment of receipts for ' 'the current event') +parser.add_argument("--receipt-endpoint", help="Attempt to connect to witness receipt endpoint for witness receipts.", + dest="endpoint", action='store_true') +parser.add_argument("--authenticate", '-z', help="Prompt the controller for authentication codes for each witness", + action='store_true') def handler(args): @@ -46,17 +51,18 @@ def handler(args): alias = args.alias force = args.force - icpDoer = InceptDoer(name=name, base=base, alias=alias, bran=bran, force=force) + subDoer = SubmitDoer(name=name, base=base, alias=alias, bran=bran, force=force, authenticate=args.authenticate, + endpoint=args.endpoint) - doers = [icpDoer] + doers = [subDoer] return doers -class InceptDoer(doing.DoDoer): +class SubmitDoer(doing.DoDoer): """ DoDoer for creating a new identifier prefix and Hab with an alias. """ - def __init__(self, name, base, alias, bran, force): + def __init__(self, name, base, alias, bran, force, endpoint=False, authenticate=False): hby = existing.setupHby(name=name, base=base, bran=bran) self.hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer @@ -64,13 +70,14 @@ def __init__(self, name, base, alias, bran, force): self.alias = alias self.hby = hby self.force = force + self.endpoint = endpoint + self.authenticate = authenticate - self.witDoer = None - doers = [self.hbyDoer, self.mbx, doing.doify(self.inceptDo)] + doers = [self.hbyDoer, self.mbx, doing.doify(self.submitDo)] - super(InceptDoer, self).__init__(doers=doers) + super(SubmitDoer, self).__init__(doers=doers) - def inceptDo(self, tymth, tock=0.0): + def submitDo(self, tymth, tock=0.0, **kwa): """ Parameters: tymth (function): injected function wrapper closure returned by .tymen() of @@ -85,18 +92,34 @@ def inceptDo(self, tymth, tock=0.0): _ = (yield self.tock) hab = self.hby.habByName(name=self.alias) - self.witDoer = agenting.WitnessReceiptor(hby=self.hby, force=self.force) - self.extend([self.witDoer]) + auths = {} + if self.authenticate: + for wit in hab.kever.wits: + code = input(f"Entire code for {wit}: ") + auths[wit] = f"{code}#{helping.nowIso8601()}" - if hab.kever.wits: - print("Waiting for witness receipts...") - self.witDoer.msgs.append(dict(pre=hab.pre)) - while not self.witDoer.cues: - _ = yield self.tock + if self.endpoint: + receiptor = agenting.Receiptor(hby=self.hby) + self.extend([receiptor]) + + yield from receiptor.receipt(hab.pre, sn=hab.kever.sn, auths=auths) + self.remove([receiptor]) + + else: + witDoer = agenting.WitnessReceiptor(hby=self.hby, force=self.force, auths=auths) + self.extend([witDoer]) + + if hab.kever.wits: + print("Waiting for witness receipts...") + witDoer.msgs.append(dict(pre=hab.pre)) + while not witDoer.cues: + _ = yield self.tock + + self.remove([witDoer]) displaying.printIdentifier(self.hby, hab.pre) - toRemove = [self.hbyDoer, self.witDoer, self.mbx] + toRemove = [self.hbyDoer, self.mbx] self.remove(toRemove) return diff --git a/src/keri/app/cli/common/displaying.py b/src/keri/app/cli/common/displaying.py index 65745f2cf..55ece7287 100644 --- a/src/keri/app/cli/common/displaying.py +++ b/src/keri/app/cli/common/displaying.py @@ -34,7 +34,7 @@ def printIdentifier(hby, pre, label="Identifier"): print("Seq No:\t{}".format(kever.sner.num)) if kever.delegated: print("Delegated Identifier") - sys.stdout.write(f" Delegator: {kever.delegator} ") + sys.stdout.write(f" Delegator: {kever.delpre} ") if seal: print(f"{terming.Colors.OKGREEN}{terming.Symbols.CHECKMARK} Anchored{terming.Colors.ENDC}") else: @@ -93,7 +93,7 @@ def printExternal(hby, pre, label="Identifier"): print("Seq No:\t{}".format(kever.sner.num)) if kever.delegated: print("Delegated Identifier") - sys.stdout.write(f" Delegator: {kever.delegator} ") + sys.stdout.write(f" Delegator: {kever.delpre} ") if anchor: print(f"{terming.Colors.OKGREEN}{terming.Symbols.CHECKMARK} Anchored{terming.Colors.ENDC}") else: diff --git a/src/keri/app/cli/common/existing.py b/src/keri/app/cli/common/existing.py index b2feba5e3..dd730df95 100644 --- a/src/keri/app/cli/common/existing.py +++ b/src/keri/app/cli/common/existing.py @@ -20,6 +20,7 @@ def setupHby(name, base="", bran=None, cf=None, temp=False): base(str): optional base directory prefix bran(str): optional passcode if the Habery was created encrypted cf (Configer): optional configuration for loading reference data + temp (bool): True means create database in /tmp Returns: Habery: the configured habery @@ -46,7 +47,8 @@ def setupHby(name, base="", bran=None, cf=None, temp=False): retries += 1 hby = habbing.Habery(name=name, base=base, bran=bran, cf=cf, free=True) break - except (kering.AuthError, ValueError): + except (kering.AuthError, ValueError) as e: + print(e) if retries >= 3: raise kering.AuthError("too many attempts") print("Valid passcode required, try again...") diff --git a/src/keri/app/cli/common/incepting.py b/src/keri/app/cli/common/incepting.py index 6a0789c91..6fe211fa3 100644 --- a/src/keri/app/cli/common/incepting.py +++ b/src/keri/app/cli/common/incepting.py @@ -28,4 +28,6 @@ def addInceptingArgs(parser): help='only allow establishment events in KEL for this prefix') parser.add_argument('--data', '-d', default=None, required=False, action="store", help='Anchor data, \'@\' allowed',) + parser.add_argument('--delpre', '-di', default=None, required=False, action="store", + help='Delegator AID for delegated identfiers',) diff --git a/src/keri/app/configing.py b/src/keri/app/configing.py index 41a807a72..1ed28b896 100644 --- a/src/keri/app/configing.py +++ b/src/keri/app/configing.py @@ -40,6 +40,35 @@ class Configer(filing.Filer): Attributes: (see Filer for inherited attributes) human (bool): True (default) means use human friendly HJSON when fext is JSON + + + + config file json or hjon + + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "nel": + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "curls": + [ + "tcp://localhost:5621/" + ] + }, + "iurls": + [ + "tcp://localhost:5620/?role=peer&name=tam" + ], + "durls": + [ + "http://127.0.0.1:7723/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy", + "http://127.0.0.1:7723/oobi/EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi", + ], + "wurls": + [ + "http://127.0.0.1:5644/.well-known/keri/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy?name=Root" + ] + } """ TailDirPath = "keri/cf" CleanTailDirPath = "keri/clean/cf" @@ -196,7 +225,7 @@ def __init__(self, configer, **kwa): super(ConfigerDoer, self).__init__(**kwa) self.configer = configer - def enter(self): + def enter(self, *, temp=None): """""" if not self.configer.opened: self.configer.reopen() diff --git a/src/keri/app/connecting.py b/src/keri/app/connecting.py index 7ab81dd13..e6a2f545c 100644 --- a/src/keri/app/connecting.py +++ b/src/keri/app/connecting.py @@ -201,7 +201,7 @@ def setImg(self, pre, typ, stream): stream (file): file-like stream of image data """ - self.hby.db.delTopVal(db=self.hby.db.imgs, key=pre.encode("utf-8")) + self.hby.db.delTopVal(db=self.hby.db.imgs, top=pre.encode("utf-8")) key = f"{pre}.content-type".encode("utf-8") self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=typ.encode("utf-8")) diff --git a/src/keri/app/delegating.py b/src/keri/app/delegating.py index ff0418ba4..252176911 100644 --- a/src/keri/app/delegating.py +++ b/src/keri/app/delegating.py @@ -19,8 +19,8 @@ logger = help.ogler.getLogger() -class Sealer(doing.DoDoer): - """ +class Anchorer(doing.DoDoer): + """Anchorer subclass of DoDoer Sends messages to Delegator of an identifier and wait for the anchoring event to be processed to ensure the inception or rotation event has been approved by the delegator. @@ -28,7 +28,7 @@ class Sealer(doing.DoDoer): """ - def __init__(self, hby, proxy=None, **kwa): + def __init__(self, hby, proxy=None, auths=None, **kwa): """ For the current event, gather the current set of witnesses, send the event, gather all receipts and send them to all other witnesses @@ -44,53 +44,36 @@ def __init__(self, hby, proxy=None, **kwa): self.postman = forwarding.Poster(hby=hby) self.witq = agenting.WitnessInquisitor(hby=hby) self.witDoer = agenting.Receiptor(hby=self.hby) + self.publishers = dict() self.proxy = proxy + self.auths = auths - super(Sealer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], - **kwa) + super(Anchorer, self).__init__(doers=[self.witq, self.witDoer, self.postman, doing.doify(self.escrowDo)], **kwa) - def delegation(self, pre, sn=None, proxy=None): + def delegation(self, pre, sn=None, proxy=None, auths=None): if pre not in self.hby.habs: raise kering.ValidationError(f"{pre} is not a valid local AID for delegation") + if proxy is not None: + self.proxy = proxy + + self.publishers[pre] = agenting.WitnessPublisher(hby=self.hby) # load the hab of the delegated identifier to anchor hab = self.hby.habs[pre] - delpre = hab.kever.delegator # get the delegator identifier + delpre = hab.kever.delpre # get the delegator identifier if delpre not in hab.kevers: raise kering.ValidationError(f"delegator {delpre} not found, unable to process delegation") - dkever = hab.kevers[delpre] # and the delegator's kever sn = sn if sn is not None else hab.kever.sner.num + self.auths = auths if auths is not None else self.auths # load the event and signatures evt = hab.makeOwnEvent(sn=sn) - smids = [] - if isinstance(hab, GroupHab): - phab = hab.mhab - smids = hab.smids - elif proxy is not None: - phab = proxy - elif hab.kever.sn > 0: - phab = hab - elif self.proxy is not None: - phab = self.proxy - else: - raise kering.ValidationError("no proxy to send messages for delegation") - # Send exn message for notification purposes - exn, atc = delegateRequestExn(phab, delpre=delpre, evt=bytes(evt), aids=smids) - - self.postman.send(hab=phab, dest=hab.kever.delegator, topic="delegate", serder=exn, attachment=atc) - srdr = serdering.SerderKERI(raw=evt) - del evt[:srdr.size] - self.postman.send(hab=phab, dest=delpre, topic="delegate", serder=srdr, attachment=evt) - - seal = dict(i=srdr.pre, s=srdr.snh, d=srdr.said) - self.witq.query(hab=phab, pre=dkever.prefixer.qb64, anchor=seal) - - self.hby.db.dune.pin(keys=(srdr.pre, srdr.said), val=srdr) + self.witDoer.msgs.append(dict(pre=pre, sn=srdr.sn, auths=self.auths)) + self.hby.db.dpwe.pin(keys=(srdr.pre, srdr.said), val=srdr) def complete(self, prefixer, seqner, saider=None): """ Check for completed delegation protocol for the specific event @@ -112,7 +95,7 @@ def complete(self, prefixer, seqner, saider=None): return True - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be compeleted. Steps involve: @@ -139,8 +122,9 @@ def escrowDo(self, tymth, tock=1.0): yield 0.5 def processEscrows(self): - self.processUnanchoredEscrow() self.processPartialWitnessEscrow() + self.processUnanchoredEscrow() + self.processWitnessPublication() def processUnanchoredEscrow(self): """ @@ -151,19 +135,19 @@ def processUnanchoredEscrow(self): """ for (pre, said), serder in self.hby.db.dune.getItemIter(): # group partial witness escrow kever = self.hby.kevers[pre] - dkever = self.hby.kevers[kever.delegator] + dkever = self.hby.kevers[kever.delpre] seal = dict(i=serder.pre, s=serder.snh, d=serder.said) - if dserder := self.hby.db.findAnchoringSealEvent(dkever.prefixer.qb64, seal=seal): + if dserder := self.hby.db.fetchLastSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal): seqner = coring.Seqner(sn=dserder.sn) couple = seqner.qb64b + dserder.saidb dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb) self.hby.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) - self.witDoer.msgs.append(dict(pre=pre, sn=serder.sn)) # Move to escrow waiting for witness receipts - print(f"Waiting for fully signed witness receipts for {serder.sn}") - self.hby.db.dpwe.pin(keys=(pre, said), val=serder) + logger.info(f"Delegation approval received, {serder.pre} confirmed, publishing to my witnesses") + self.publishDelegator(pre) + self.hby.db.dpub.put(keys=(pre, said), val=serder) self.hby.db.dune.rem(keys=(pre, said)) def processPartialWitnessEscrow(self): @@ -188,9 +172,75 @@ def processPartialWitnessEscrow(self): witnessed = True if not witnessed: continue - print(f"Witness receipts complete, {pre} confirmed.") + logger.info(f"Witness receipts complete, waiting for delegation approval.") + if pre not in self.hby.habs: + continue + + hab = self.hby.habs[pre] + delpre = hab.kever.delpre # get the delegator identifier + dkever = hab.kevers[delpre] # and the delegator's kever + smids = [] + + if isinstance(hab, GroupHab): + phab = hab.mhab + smids = hab.smids + elif self.proxy is not None: + phab = self.proxy + else: + raise kering.ValidationError("no proxy to send messages for delegation") + + evt = hab.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said) + srdr = serdering.SerderKERI(raw=evt) + exn, atc = delegateRequestExn(phab, delpre=delpre, evt=bytes(evt), aids=smids) + + logger.info("Sending delegation request exn for %s from %s to delegator %s", + srdr.ilk, phab.pre, delpre) + logger.debug("Delegation request=\n%s\n", exn.pretty()) + self.postman.send(hab=phab, dest=hab.kever.delpre, topic="delegate", serder=exn, attachment=atc) + + del evt[:srdr.size] + logger.info("Sending delegation event %s from %s to delegator %s", + srdr.ilk, phab.pre, delpre) + logger.debug("Delegated inception=\n%s\n", srdr.pretty()) + self.postman.send(hab=phab, dest=delpre, topic="delegate", serder=srdr, attachment=evt) + + seal = dict(i=srdr.pre, s=srdr.snh, d=srdr.said) + self.witq.query(hab=phab, pre=dkever.prefixer.qb64, anchor=seal) + self.hby.db.dpwe.rem(keys=(pre, said)) - self.hby.db.cdel.put(keys=(pre, seqner.qb64), val=coring.Saider(qb64=serder.said)) + self.hby.db.dune.pin(keys=(srdr.pre, srdr.said), val=srdr) + + def processWitnessPublication(self): + """ + Process escrow of partially signed multisig group KEL events. Message + processing will send this local controllers signature to all other participants + then this escrow waits for signatures from all other participants + + """ + for (pre, said), serder in self.hby.db.dpub.getItemIter(): # group partial witness escrow + if pre not in self.publishers: + continue + + publisher = self.publishers[pre] + + if not publisher.idle: + continue + + self.remove([publisher]) + del self.publishers[pre] + + self.hby.db.dpub.rem(keys=(pre, said)) + self.hby.db.cdel.put(keys=(pre, coring.Seqner(sn=serder.sn).qb64), val=coring.Saider(qb64=serder.said)) + + def publishDelegator(self, pre): + if pre not in self.publishers: + return + + publisher = self.publishers[pre] + hab = self.hby.habs[pre] + self.extend([publisher]) + for msg in hab.db.cloneDelegation(hab.kever): + publisher.msgs.append(dict(pre=hab.pre, msg=bytes(msg))) def loadHandlers(hby, exc, notifier): diff --git a/src/keri/app/directing.py b/src/keri/app/directing.py index 70bd96313..5da16998c 100644 --- a/src/keri/app/directing.py +++ b/src/keri/app/directing.py @@ -8,6 +8,7 @@ import itertools from hio.base import doing +from . import tocking from .. import help from ..core import eventing, routing from ..core import parsing @@ -207,7 +208,7 @@ def wind(self, tymth): self.client.wind(tymth) - def msgDo(self, tymth=None, tock=0.0, **opts): + def msgDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -227,14 +228,16 @@ def msgDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactorMsgTock + _ = (yield self.tock) # enter context if self.parser.ims: logger.info("Client %s received:\n%s\n...\n", self.hab.name, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def cueDo(self, tymth=None, tock=0.0, **opts): + def cueDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -253,15 +256,17 @@ def cueDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactorCueTock + _ = (yield self.tock) # enter context while True: for msg in self.hab.processCuesIter(self.kevery.cues): self.sendMessage(msg, label="chit or receipt") - yield # throttle just do one cue at a time - yield + yield self.tock # throttle just do one cue at a time + yield self.tock return False # should never get here except forced close - def escrowDo(self, tymth=None, tock=0.0, **opts): + def escrowDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -280,12 +285,14 @@ def escrowDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactorEscrowTock + _ = (yield self.tock) # enter context while True: self.kevery.processEscrows() if self.tvy is not None: self.tvy.processEscrows() - yield + yield self.tock return False # should never get here except forced close def sendMessage(self, msg, label=""): @@ -561,7 +568,7 @@ def wind(self, tymth): self.remoter.wind(tymth) - def msgDo(self, tymth=None, tock=0.0, **opts): + def msgDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -581,15 +588,17 @@ def msgDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactantMsgTock + _ = (yield self.tock) # enter context if self.parser.ims: logger.info("Server %s: received:\n%s\n...\n", self.hab.name, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def cueDo(self, tymth=None, tock=0.0, **opts): + def cueDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -608,19 +617,21 @@ def cueDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactantCueTock + _ = (yield self.tock) # enter context while True: for msg in self.hab.processCuesIter(self.kevery.cues): if isinstance(msg, list): msg = bytearray(itertools.chain(*msg)) self.sendMessage(msg, label="chit or receipt or replay") - yield # throttle just do one cue at a time - yield + yield self.tock # throttle just do one cue at a time + yield self.tock return False # should never get here except forced close - def escrowDo(self, tymth=None, tock=0.0, **opts): + def escrowDo(self, tymth=None, tock=None, **opts): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -639,12 +650,14 @@ def escrowDo(self, tymth=None, tock=0.0, **opts): Usage: add result of doify on this method to doers list """ - yield # enter context + self.wind(tymth) + self.tock = tock if tock is not None else tocking.ReactantEscrowTock + _ = (yield self.tock) # enter context while True: self.kevery.processEscrows() if self.tevery is not None: self.tevery.processEscrows() - yield + yield self.tock return False # should never get here except forced close def sendMessage(self, msg, label=""): diff --git a/src/keri/app/forwarding.py b/src/keri/app/forwarding.py index b4105b2a1..fda62ebdc 100644 --- a/src/keri/app/forwarding.py +++ b/src/keri/app/forwarding.py @@ -9,18 +9,17 @@ from ordered_set import OrderedSet as oset from hio.base import doing -from hio.help import decking +from hio.help import decking, ogler from keri import kering from keri.app import agenting from keri.app.habbing import GroupHab -from keri import help from keri.core import coring, eventing, serdering from keri.db import dbing from keri.kering import Roles from keri.peer import exchanging -logger = help.ogler.getLogger() +logger = ogler.getLogger() class Poster(doing.DoDoer): @@ -40,7 +39,7 @@ def __init__(self, hby, mbx=None, evts=None, cues=None, **kwa): doers = [doing.doify(self.deliverDo)] super(Poster, self).__init__(doers=doers, **kwa) - def deliverDo(self, tymth=None, tock=0.0): + def deliverDo(self, tymth=None, tock=0.0, **kwa): """ Returns: doifiable Doist compatible generator method that processes a queue of messages and envelopes them in a `fwd` message @@ -86,7 +85,7 @@ def deliverDo(self, tymth=None, tock=0.0): elif Roles.witness in ends: yield from self.forwardToWitness(hab, ends[Roles.witness], recp=recp, serder=srdr, atc=atc, topic=tpc) else: - logger.info(f"No end roles for {recp} to send evt={recp}") + logger.info(f"No end roles for {recp} to send evt={srdr.said}") continue except kering.ConfigurationError as e: logger.error(f"Error sending to {recp} with ends={ends}. Err={e}") @@ -136,15 +135,14 @@ def sent(self, said): return False - def sendEvent(self, hab, fn=0): + def sendEventToDelegator(self, sender, hab, fn=0): """ Returns generator for sending event and waiting until send is complete """ # Send KEL event for processing icp = self.hby.db.cloneEvtMsg(pre=hab.pre, fn=fn, dig=hab.kever.serder.saidb) ser = serdering.SerderKERI(raw=icp) del icp[:ser.size] - sender = hab.mhab.pre if isinstance(hab, GroupHab) else hab.pre - self.send(src=sender, dest=hab.kever.delegator, topic="delegate", serder=ser, attachment=icp) + self.send(src=sender.pre, dest=hab.kever.delpre, topic="delegate", serder=ser, attachment=icp) while True: if self.cues: cue = self.cues.popleft() @@ -452,7 +450,7 @@ def handle(self, serder, attachments=None): pevt = bytearray() for pather, atc in attachments: ked = pather.resolve(embeds) - sadder = coring.Sadder(ked=ked, kind=eventing.Serials.json) + sadder = coring.Sadder(ked=ked, kind=eventing.Kinds.json) pevt.extend(sadder.raw) pevt.extend(atc) diff --git a/src/keri/app/grouping.py b/src/keri/app/grouping.py index 51f58ce0a..8b748a982 100644 --- a/src/keri/app/grouping.py +++ b/src/keri/app/grouping.py @@ -9,10 +9,10 @@ from hio.base import doing from hio.help import decking -from .. import kering +from .. import kering, core from .. import help from ..app import delegating, agenting -from ..core import coring, routing, eventing, parsing, serdering +from ..core import coring, routing, eventing, parsing, serdering, indexing from ..db import dbing from ..db.dbing import snKey from ..peer import exchanging @@ -25,7 +25,7 @@ class Counselor(doing.DoDoer): def __init__(self, hby, swain=None, proxy=None, **kwa): self.hby = hby - self.swain = swain if swain is not None else delegating.Sealer(hby=self.hby) + self.swain = swain if swain is not None else delegating.Anchorer(hby=self.hby) self.proxy = proxy self.witDoer = agenting.Receiptor(hby=self.hby) self.witq = agenting.WitnessInquisitor(hby=hby) @@ -47,11 +47,9 @@ def start(self, ghab, prefixer, seqner, saider): saider (Saider): saider of event of group identifier """ - evt = ghab.makeOwnEvent(sn=seqner.sn, allowPartiallySigned=True) - serder = serdering.SerderKERI(raw=evt) - del evt[:serder.size] - - logger.info(f"Waiting for other signatures for {serder.ilk} {serder.pre}:{seqner.sn}...") + evt = ghab.makeOwnEvent(sn=seqner.sn, allowPartiallySigned=True) # used just for the log message + serder = serdering.SerderKERI(raw=evt) # used just for the log message + logger.info("Waiting for other signatures on %s for %s:%s...", serder.ilk, prefixer.qb64, seqner.sn) return self.hby.db.gpse.add(keys=(prefixer.qb64,), val=(seqner, saider)) def complete(self, prefixer, seqner, saider=None): @@ -74,7 +72,7 @@ def complete(self, prefixer, seqner, saider=None): return True - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be compeleted. Steps involve: @@ -124,7 +122,7 @@ def processPartialSignedEscrow(self): if not sigs: # otherwise its a list of sigs continue - sigers = [coring.Siger(qb64b=bytes(sig)) for sig in sigs] + sigers = [indexing.Siger(qb64b=bytes(sig)) for sig in sigs] windex = min([siger.index for siger in sigers]) # True if Elected to perform delegation and witnessing @@ -133,24 +131,27 @@ def processPartialSignedEscrow(self): if kever.delegated and kever.ilk in (coring.Ilks.dip, coring.Ilks.drt): # We are a delegated identifier, must wait for delegator approval for dip and drt if witered: # We are elected to perform delegation and witnessing messaging - logger.info(f"We are the witnesser, sending {pre} to delegator") + logger.info("AID %s...%s: We are the witnesser, sending %s to delegator", + pre[:4], pre[-4:], pre) self.swain.delegation(pre=pre, sn=seqner.sn) else: anchor = dict(i=pre, s=seqner.snh, d=saider.qb64) if self.proxy: - self.witq.query(hab=self.proxy, pre=kever.delegator, anchor=anchor) + self.witq.query(hab=self.proxy, pre=kever.delpre, anchor=anchor) else: - self.witq.query(src=ghab.mhab.pre, pre=kever.delegator, anchor=anchor) + self.witq.query(src=ghab.mhab.pre, pre=kever.delpre, anchor=anchor) - logger.info(f"Waiting for delegation approval...") + logger.info("AID %s...%s: Waiting for delegation approval...", pre[:4], pre[-4:]) self.hby.db.gdee.add(keys=(pre,), val=(seqner, saider)) else: # Non-delegation, move on to witnessing if witered: # We are elected witnesser, send off event to witnesses - logger.info(f"We are the fully signed witnesser {seqner.sn}, sending to witnesses") + logger.info("AID %s...%s: We are the fully signed witnesser %s, sending to witnesses", + pre[:4], pre[-4:], seqner.sn) self.witDoer.msgs.append(dict(pre=pre, sn=seqner.sn)) # Move to escrow waiting for witness receipts - logger.info(f"Waiting for fully signed witness receipts for {seqner.sn}") + logger.info("AID %s...%s: Waiting for fully signed witness receipts for %s", + pre[:4], pre[-4:], seqner.sn) self.hby.db.gpwe.add(keys=(pre,), val=(seqner, saider)) def processDelegateEscrow(self): @@ -170,21 +171,23 @@ def processDelegateEscrow(self): if witer: # We are elected witnesser, We've already done out part in Boatswain, we are done. if self.swain.complete(prefixer=kever.prefixer, seqner=coring.Seqner(sn=kever.sn)): self.hby.db.gdee.rem(keys=(pre,)) - logger.info(f"Delegation approval for {pre} received.") + logger.info("AID %s...%s: Delegation approval for %s received.", + pre[:4], pre[-4:], pre) self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) else: # Not witnesser, we need to look for the anchor and then wait for receipts - if serder := self.hby.db.findAnchoringSealEvent(kever.delegator, seal=anchor): + if serder := self.hby.db.fetchLastSealingEventByEventSeal(kever.delpre, + seal=anchor): aseq = coring.Seqner(sn=serder.sn) couple = aseq.qb64b + serder.saidb dgkey = dbing.dgKey(pre, saider.qb64b) self.hby.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) self.hby.db.gdee.rem(keys=(pre,)) - logger.info(f"Delegation approval for {pre} received.") + logger.info("AID %s...%s: Delegation approval for %s received.", pre[:4], pre[-4:], pre) # Move to escrow waiting for witness receipts - logger.info(f"Waiting for witness receipts for {pre}") + logger.info("AID %s...%s: Waiting for witness receipts for %s", pre[:4], pre[-4:], pre) self.hby.db.gdee.rem(keys=(pre,)) self.hby.db.gpwe.add(keys=(pre,), val=(seqner, saider)) @@ -212,7 +215,7 @@ def processPartialWitnessEscrow(self): witnessed = True if not witnessed: continue - logger.info(f"Witness receipts complete, {pre} confirmed.") + logger.info("AID %s...%s: Witness receipts complete, %s confirmed.", pre[:4], pre[-4:], pre) self.hby.db.gpwe.rem(keys=(pre,)) self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider) elif not witer: @@ -243,7 +246,8 @@ def handle(self, serder, attachments=None): attachments (list): list of tuples of pather, CESR SAD path attachments to the exn event """ - logger.info("handling %s event SAID=%s", self.resource, serder.said) + logger.info("Notification for %s event SAID=%s", self.resource, serder.said) + logger.debug("EXN Body=\n%s\n", serder.pretty()) self.mux.add(serder=serder) @@ -280,7 +284,7 @@ def multisigInceptExn(hab, smids, rmids, icp, delegator=None): """ rmids = rmids if rmids is not None else smids - serder = serdering.SerderKERI(raw=icp) # coring.Serder(raw=icp) + serder = serdering.SerderKERI(raw=icp) data = dict( gid=serder.pre, smids=smids, @@ -517,20 +521,20 @@ def getEscrowedEvent(db, pre, sn): sigs = [] for sig in db.getSigsIter(key): - sigs.append(coring.Siger(qb64b=bytes(sig))) + sigs.append(indexing.Siger(qb64b=bytes(sig))) couple = db.getAes(key) msg = bytearray() msg.extend(serder.raw) - msg.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigs)).qb64b) # attach cnt + msg.extend(core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigs), gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt for sig in sigs: msg.extend(sig.qb64b) # attach sig if couple is not None: - msg.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + msg.extend(core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(couple) return msg @@ -590,7 +594,7 @@ def add(self, serder): events so that any addition signatures can be processed. Parameters: - serder (coring.Serder): peer-to-peer exn "/multisig" message to coordinate from other participants + serder (serdering.SerderKERI): peer-to-peer exn "/multisig" message to coordinate from other participants Returns: @@ -667,7 +671,7 @@ def add(self, serder): ims.extend(atc) # ... and parse - self.psr.parse(ims=ims) + self.psr.parse(ims=ims, local=True) else: # Should we prod the user with another submission if we haven't already approved it? diff --git a/src/keri/app/habbing.py b/src/keri/app/habbing.py index d30f143e3..fc11d244f 100644 --- a/src/keri/app/habbing.py +++ b/src/keri/app/habbing.py @@ -16,7 +16,9 @@ from . import keeping, configing from .. import help from .. import kering -from ..core import coring, eventing, parsing, routing, serdering +from .. import core +from ..core import (coring, eventing, parsing, routing, serdering, indexing, + Counter, Codens) from ..db import dbing, basing from ..kering import MissingSignatureError, Roles @@ -57,9 +59,10 @@ def openHby(*, name="test", base="", temp=True, salt=None, **kwa): all secrets are re-encrypted using new aeid. In this case the provided prikey must not be empty. A change in aeid should require a second authentication mechanism besides the prikey. - bran (str): Base64 22 char string that is used as base material for - seed. bran allows alphanumeric passcodes generated by key managers - like 1password to be key store for seed. + bran (str): Base64 char string of which first 21 chars are used as + base material for seed. bran allows alphanumeric passcodes + generated by key managers like 1password to be key store for + bran as salt base material for seed. pidx (int): Initial prefix index for vacuous ks algo (str): algorithm (randy or salty) for creating key pairs default is root algo which defaults to salty @@ -68,7 +71,7 @@ def openHby(*, name="test", base="", temp=True, salt=None, **kwa): """ habery = None - salt = salt if not None else coring.Salter(raw=b'0123456789abcdef').qb64 + salt = salt if salt is not None else core.Salter().qb64 try: habery = Habery(name=name, base=base, temp=temp, salt=salt, **kwa) yield habery @@ -79,7 +82,7 @@ def openHby(*, name="test", base="", temp=True, salt=None, **kwa): @contextmanager -def openHab(name="test", base="", salt=b'0123456789abcdef', temp=True, cf=None, **kwa): +def openHab(name="test", base="", salt=None, temp=True, cf=None, **kwa): """ Context manager wrapper for Hab instance. Defaults to temporary resources @@ -95,11 +98,11 @@ def openHab(name="test", base="", salt=b'0123456789abcdef', temp=True, cf=None, """ - salt = coring.Salter(raw=salt).qb64 + salt = core.Salter(raw=salt).qb64 with openHby(name=name, base=base, salt=salt, temp=temp, cf=cf) as hby: if (hab := hby.habByName(name)) is None: - hab = hby.makeHab(name=name, icount=1, isith='1', ncount=1, nsith='1', **kwa) + hab = hby.makeHab(name=name, icount=1, isith='1', ncount=1, nsith='1', cf=cf, **kwa) yield hby, hab @@ -129,8 +132,10 @@ class Habery: psr (parsing.Parser): parses local messages for .kvy .rvy habs (dict): Hab instances keyed by prefix. - To look up Hab by name get prefix from db.habs .prefix field using - .habByName + To look up Hab by name use use .habByName + To look up Hab by prefix us .habByPrefix + to get hab from db need name for key + hab prefix in db.habs record .hid field inited (bool): True means fully initialized wrt databases. False means not yet fully initialized @@ -182,9 +187,10 @@ def __init__(self, *, name='test', base="", temp=False, all secrets Haberyare re-encrypted using new aeid. In this case the provided prikey must not be empty. A change in aeid should require a second authentication mechanism besides the prikey. - bran (str): Base64 22 char string that is used as base material for - seed. bran allows alphanumeric passcodes generated by key managers - like 1password to be key store for seed. + bran (str): Base64 char string of which first 21 chars are used as + base material for seed. bran allows alphanumeric passcodes + generated by key managers like 1password to be key store for + bran as salt base material for seed. pidx (int): Initial prefix index for vacuous ks algo (str): algorithm (randy or salty) for creating key pairs default is root algo which defaults to salty @@ -221,9 +227,9 @@ def __init__(self, *, name='test', base="", temp=False, self.exc = exchanging.Exchanger(hby=self, handlers=[]) self.kvy = eventing.Kevery(db=self.db, lax=False, local=True, rvy=self.rvy) self.kvy.registerReplyRoutes(router=self.rtr) - self.psr = parsing.Parser(framed=True, kvy=self.kvy, rvy=self.rvy, exc=self.exc) + self.psr = parsing.Parser(framed=True, kvy=self.kvy, rvy=self.rvy, + exc=self.exc, local=True) self.habs = {} # empty .habs - self.namespaces = {} # empty .namespaces self._signator = None self.inited = False @@ -235,6 +241,7 @@ def __init__(self, *, name='test', base="", temp=False, if self.db.opened and self.ks.opened: self.setup(**self._inits) # finish setup later + def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, salt=None, tier=None, free=False, temp=None, ): """ @@ -285,7 +292,7 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, if len(bran) < 21: raise ValueError(f"Bran (passcode seed material) too short.") bran = coring.MtrDex.Salt_128 + 'A' + bran[:21] # qb64 salt for seed - signer = coring.Salter(qb64=bran).signer(transferable=False, + signer = core.Salter(qb64=bran).signer(transferable=False, tier=tier, temp=temp) seed = signer.qb64 @@ -293,9 +300,9 @@ def setup(self, *, seed=None, aeid=None, bran=None, pidx=None, algo=None, aeid = signer.verfer.qb64 # lest it remove encryption if salt is None: # salt for signing keys not aeid seed - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter().qb64 else: - salt = coring.Salter(qb64=salt).qb64 + salt = core.Salter(qb64=salt).qb64 try: self.mgr = keeping.Manager(ks=self.ks, seed=seed, aeid=aeid, pidx=pidx, @@ -322,79 +329,40 @@ def loadHabs(self): self.reconfigure() # pre hab load reconfiguration groups = [] - for name, habord in self.db.habs.getItemIter(): - name = ".".join(name) # detupleize the database key name + for prefix, habord in self.db.habs.getItemIter(): pre = habord.hid # create Hab instance and inject dependencies if habord.mid and not habord.sid: hab = GroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre, temp=self.temp, smids=habord.smids, rmids=habord.rmids) + name=habord.name, pre=pre, temp=self.temp, smids=habord.smids) groups.append(habord) elif habord.sid and not habord.mid: hab = SignifyHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=habord.sid) + name=habord.name, pre=habord.sid) elif habord.sid and habord.mid: - hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, + hab = SignifyGroupHab(smids=habord.smids, ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre) + name=habord.name, pre=pre) groups.append(habord) else: hab = Hab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=pre, temp=self.temp) + name=habord.name, pre=pre, temp=self.temp) - # Rules for acceptance - # if its delegated its accepted into its own local KEL even if the - # delegator has not sealed it + # Rules for acceptance: + # It is accepted into its own local KEL even if it has not been fully + # witnessed and if delegated, its delegator has not yet sealed it if not hab.accepted and not habord.mid: raise kering.ConfigurationError(f"Problem loading Hab pre=" - f"{pre} name={name} from db.") + f"{pre} name={habord.name} from db.") # read in config file and process any oobis or endpoints for hab hab.inited = True self.habs[hab.pre] = hab - for keys, habord in self.db.nmsp.getItemIter(): - ns = keys[0] - name = ".".join(keys[1:]) # detupleize the database key name - pre = habord.hid - - # create Hab instance and inject dependencies - if habord.mid and not habord.sid: - hab = GroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=pre, temp=self.temp, smids=habord.smids) - groups.append(habord) - elif habord.sid and not habord.mid: - hab = SignifyHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=habord.sid) - elif habord.sid and habord.mid: - hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, pre=habord.sid) - groups.append(habord) - else: - hab = Hab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, - rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, ns=ns, pre=pre, temp=self.temp) - - # Rules for acceptance - # if its delegated its accepted into its own local KEL even if the - # delegator has not sealed it - if not hab.accepted and not habord.mid: - raise kering.ConfigurationError(f"Problem loading Hab pre=" - f"{pre} name={name} from db.") - - # read in config file and process any oobis or endpoints for hab - hab.inited = True - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - # Populate the participant hab after loading all habs for habord in groups: self.habs[habord.hid].mhab = self.habs[habord.mid] @@ -417,7 +385,7 @@ def makeHab(self, name, ns=None, cf=None, **kwa): toad (Union[int,str]): int or str hex of witness threshold wits (list): of qb64 prefixes of witnesses delpre (str): qb64 of delegator identifier prefix - estOnly (str): eventing.TraitCodex.EstOnly means only establishment + estOnly (str): eventing.TraitDex.EstOnly means only establishment events allowed in KEL for this Hab data (list | None): seal dicts """ @@ -431,13 +399,7 @@ def makeHab(self, name, ns=None, cf=None, **kwa): hab.make(**kwa) - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): @@ -469,9 +431,9 @@ def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): toad (Union[int,str]): int or str hex of witness threshold wits (list): of qb64 prefixes of witnesses delpre (str): qb64 of delegator identifier prefix - estOnly (str): eventing.TraitCodex.EstOnly means only establishment + estOnly (str): eventing.TraitDex.EstOnly means only establishment events allowed in KEL for this Hab - DnD (bool): eventing.TraitCodex.DnD means do allow delegated identifiers from this identifier + DnD (bool): eventing.TraitDex.DnD means do allow delegated identifiers from this identifier ToDo: NRR add midxs tuples for each group member or all in group multisig. @@ -508,13 +470,7 @@ def makeGroupHab(self, group, mhab, smids, rmids=None, ns=None, **kwa): name=group, ns=ns, mhab=mhab, smids=smids, rmids=rmids, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): @@ -562,6 +518,8 @@ def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, + name=hab.name, + domain=ns, mid=mhab.pre, smids=smids, rmids=rmids) @@ -570,13 +528,7 @@ def joinGroupHab(self, pre, group, mhab, smids, rmids=None, ns=None): hab.prefixes.add(pre) hab.inited = True - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab def makeSignifyHab(self, name, ns=None, **kwa): @@ -586,29 +538,18 @@ def makeSignifyHab(self, name, ns=None, **kwa): name=name, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab - def makeSignifyGroupHab(self, name, mhab, ns=None, **kwa): + def makeSignifyGroupHab(self, name, mhab, smids, rmids=None, ns=None, **kwa): # create group Hab in this Habery hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, mhab=mhab, ns=ns, temp=self.temp) + name=name, mhab=mhab, smids=smids, rmids=rmids, ns=ns, temp=self.temp) hab.make(**kwa) # finish making group hab with injected pass throughs - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab + self.habs[hab.pre] = hab return hab def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): @@ -652,11 +593,13 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): # create group Hab in this Habery hab = SignifyGroupHab(ks=self.ks, db=self.db, cf=self.cf, mgr=self.mgr, rtr=self.rtr, rvy=self.rvy, kvy=self.kvy, psr=self.psr, - name=name, mhab=mhab, ns=ns, temp=self.temp) + name=name, mhab=mhab, smids=smids, rmids=rmids, ns=ns, temp=self.temp) hab.pre = pre habord = basing.HabitatRecord(hid=hab.pre, sid=mhab.pre, + name=name, + domain=ns, smids=smids, rmids=rmids) @@ -664,25 +607,25 @@ def joinSignifyGroupHab(self, pre, name, mhab, smids, rmids=None, ns=None): hab.prefixes.add(pre) hab.inited = True - if ns is None: - self.habs[hab.pre] = hab - else: - if ns not in self.namespaces: - self.namespaces[ns] = dict() - self.namespaces[ns][hab.pre] = hab - + self.habs[hab.pre] = hab return hab - def deleteHab(self, name): - hab = self.habByName(name) + def deleteHab(self, name, ns=None): + hab = self.habByName(name, ns=ns) if not hab: return False - if not self.db.habs.rem(keys=(name,)): + if not self.db.habs.rem(keys=(hab.pre,)): + return False + + ns = "" if ns is None else ns + if not self.db.names.rem(keys=(ns, name)): return False del self.habs[hab.pre] self.db.prefixes.remove(hab.pre) + if hab.pre in self.db.groups: + self.db.groups.remove(hab.pre) return True @@ -752,7 +695,8 @@ def prefixes(self): def habByPre(self, pre): """ - Returns the Hab from and namespace including the default namespace. + Returns the Hab instance from .habs or None + including the default namespace. Args: pre (str): qb64 aid of hab to find @@ -760,33 +704,26 @@ def habByPre(self, pre): Returns: Hab: Hab instance for the aid pre or None """ - hab = None if pre in self.habs: - hab = self.habs[pre] - else: - for nsp in self.namespaces.values(): - if pre in nsp: - hab = nsp[pre] + return self.habs[pre] - return hab + return None def habByName(self, name, ns=None): """ Returns: - hab (Hab): instance from .habs by name if any otherwise None + hab (Hab): instance by name from .habs or .namspaces + if any otherwise None Parameters: name (str): alias of Hab ns (str): optional namespace of hab """ - if ns is not None: - if (habord := self.db.nmsp.get(keys=(ns, name))) is not None: - habs = self.namespaces[ns] - return habs[habord.hid] if habord.hid in habs else None - - elif (habord := self.db.habs.get(name)) is not None: - return self.habs[habord.hid] if habord.hid in self.habs else None + ns = "" if ns is None else ns + if (pre := self.db.names.get(keys=(ns, name))) is not None: + if pre in self.habs: + return self.habs[pre] return None @@ -794,13 +731,34 @@ def reconfigure(self): """Apply configuration from config file managed by .cf. to this Habery Process any oobis or endpoints - conf + config file json or hjon + { - dt: "isodatetime", - curls: ["tcp://localhost:5620/"], - iurls: ["tcp://localhost:5621/?name=eve"], + "dt": "2021-01-01T00:00:00.000000+00:00", + "nel": + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "curls": + [ + "tcp://localhost:5621/" + ] + }, + "iurls": + [ + "tcp://localhost:5620/?role=peer&name=tam" + ], + "durls": + [ + "http://127.0.0.1:7723/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy", + "http://127.0.0.1:7723/oobi/EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi", + ], + "wurls": + [ + "http://127.0.0.1:5644/.well-known/keri/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy?name=Root" + ] } + Config file is meant to be read only at init not changed by app at run time. Any dynamic app changes must go in database not config file that way we don't have to worry about multiple writers of cf. @@ -942,7 +900,7 @@ def __init__(self, habery, **kwa): super(HaberyDoer, self).__init__(**kwa) self.habery = habery - def enter(self): + def enter(self, *, temp=False): """ Enter context and set up Habery """ if not self.habery.inited: self.habery.setup(**self.habery._inits) @@ -985,7 +943,7 @@ class BaseHab: kevers (dict): of eventing.Kever instances from KELs in local db keyed by qb64 prefix. Read through cache of of kevers of states for KELs in db.states - iserder (coring.Serder): own inception event + iserder (serdering.SerderKERI): own inception event prefixes (OrderedSet): local prefixes for .db accepted (bool): True means accepted into local KEL. False otherwise @@ -1026,7 +984,7 @@ def __init__(self, ks, db, cf, mgr, rtr, rvy, kvy, psr, *, self.psr = psr # injected self.name = name - self.ns = ns + self.ns = ns # what is this? self.pre = pre # wait to setup until after db is known to be opened self.temp = True if temp else False @@ -1052,10 +1010,10 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, specified else compute default based on number of wits (backers) wits (list | None): qb64 prefixes of witnesses if any delpre (str | None): qb64 of delegator identifier prefix if any - estOnly (bool | None): True means add trait eventing.TraitCodex.EstOnly + estOnly (bool | None): True means add trait eventing.TraitDex.EstOnly which means only establishment events allowed in KEL for this Hab False (default) means allows non-est events and no trait is added. - DnD (bool): True means add trait of eventing.TraitCodex.DnD which + DnD (bool): True means add trait of eventing.TraitDex.DnD which means do not allow delegated identifiers from this identifier False (default) means do allow and no trait is added. @@ -1072,9 +1030,9 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, nst = coring.Tholder(sith=nsith).sith # next signing threshold cnfg = [] if estOnly: - cnfg.append(eventing.TraitCodex.EstOnly) + cnfg.append(eventing.TraitDex.EstOnly) if DnD: - cnfg.append(eventing.TraitCodex.DoNotDelegate) + cnfg.append(eventing.TraitDex.DoNotDelegate) self.delpre = delpre keys = [verfer.qb64 for verfer in verfers] if self.delpre: @@ -1100,25 +1058,49 @@ def make(self, DnD, code, data, delpre, estOnly, isith, verfers, nsith, digers, return serder def save(self, habord): - if self.ns is None: - self.db.habs.pin(keys=self.name, - val=habord) - else: - self.db.nmsp.put(keys=(self.ns, self.name), - val=habord) + self.db.habs.pin(keys=self.pre, + val=habord) + ns = "" if self.ns is None else self.ns + if self.db.names.get(keys=(ns, self.name)) is not None: + raise ValueError("AID already exists with that name") + + self.db.names.pin(keys=(ns, self.name), + val=self.pre) def reconfigure(self): """Apply configuration from config file managed by .cf. to this Hab. Assumes that .pre and signing keys have been setup in order to create own endpoint auth when provided in .cf. - conf + config file json or hjon + { - dt: "isodatetime", - curls: ["tcp://localhost:5620/"], - iurls: ["tcp://localhost:5621/?name=eve"] + "dt": "2021-01-01T00:00:00.000000+00:00", + "nel": + { + "dt": "2021-01-01T00:00:00.000000+00:00", + "curls": + [ + "tcp://localhost:5621/" + ] + }, + "iurls": + [ + "tcp://localhost:5620/?role=peer&name=tam" + ], + "durls": + [ + "http://127.0.0.1:7723/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy", + "http://127.0.0.1:7723/oobi/EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi", + ], + "wurls": + [ + "http://127.0.0.1:5644/.well-known/keri/oobi/EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy?name=Root" + ] } + + Config file is meant to be read only at init not changed by app at run time. Any dynamic app changes must go in database not config file that way we don't have to worry about multiple writers of cf. @@ -1239,7 +1221,7 @@ def rotate(self, *, verfers=None, digers=None, isith=None, nsith=None, toad=None if not kever.ntholder.satisfy(indices): raise kering.ValidationError("invalid rotation, new key set unable to satisfy prior next signing threshold") - if kever.delegator is not None: # delegator only shows up in delcept + if kever.delpre is not None: # delegator only shows up in delcept serder = eventing.deltate(pre=kever.prefixer.qb64, keys=keys, dig=kever.serder.said, @@ -1338,14 +1320,16 @@ def sign(self, ser, verfers=None, indexed=True, indices=None, ondices=None, **kw indices=indices, ondices=ondices) + def decrypt(self, ser, verfers=None, **kwa): - """Sign given serialization ser using appropriate keys. + """Decrypt given serialization ser using appropriate keys. Use provided verfers or .kever.verfers to lookup keys to sign. Parameters: - ser (bytes): serialization to sign + ser (str | bytes | bytearray | memoryview): serialization to decrypt + verfers (list[Verfer] | None): Verfer instances to get pub verifier - keys to lookup private siging keys. + keys to lookup and convert to private decryption keys. verfers None means use .kever.verfers. Assumes that when group and verfers is not None then provided verfers must be .kever.verfers @@ -1353,9 +1337,10 @@ def decrypt(self, ser, verfers=None, **kwa): if verfers is None: verfers = self.kever.verfers # when group these provide group signing keys - return self.mgr.decrypt(ser=ser, - verfers=verfers, - ) + # should not use mgr.decrypt since it assumes qb64. Just lucky its not + # yet a problem + return self.mgr.decrypt(qb64=ser, verfers=verfers) + def query(self, pre, src, query=None, **kwa): """ Create, sign and return a `qry` message against the attester for the prefix @@ -1493,8 +1478,10 @@ def witness(self, serder): indexed receipt signatures if key state of serder.pre shows that own pre is a current witness of event in serder - Before calling this must check that serder being witnessed has been - accepted as valid event into controller's KEL + ToDo XXXX add parameter to force check that serder as been accepted + as valid. Otherwise must assume that before calling this that serder + being witnessed has been accepted as valid event into this hab + controller's KEL """ if self.kever.prefixer.transferable: # not non-transferable prefix @@ -1551,7 +1538,7 @@ def replay(self, pre=None, fn=0): return msgs - def replayAll(self, key=b''): + def replayAll(self): """ Returns replay of FEL first seen event log for all pre starting at key @@ -1560,7 +1547,7 @@ def replayAll(self, key=b''): """ msgs = bytearray() - for msg in self.db.cloneAllPreIter(key=key): + for msg in self.db.cloneAllPreIter(): msgs.extend(msg) return msgs @@ -1584,8 +1571,8 @@ def makeOtherEvent(self, pre, sn): dig = bytes(dig) key = dbing.dgKey(pre, dig) # digest key msg.extend(self.db.getEvt(key)) - msg.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=self.db.cntSigs(key)).qb64b) # attach cnt + msg.extend(Counter(Codens.ControllerIdxSigs, count=self.db.cntSigs(key), + gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt for sig in self.db.getSigsIter(key): msg.extend(sig) # attach sig return msg @@ -2031,7 +2018,7 @@ def getOwnEvent(self, sn, allowPartiallySigned=False): sigs = [] for sig in self.db.getSigsIter(key): - sigs.append(coring.Siger(qb64b=bytes(sig))) + sigs.append(indexing.Siger(qb64b=bytes(sig))) couple = self.db.getAes(key) @@ -2052,14 +2039,14 @@ def makeOwnEvent(self, sn, allowPartiallySigned=False): serder, sigs, couple = self.getOwnEvent(sn=sn, allowPartiallySigned=allowPartiallySigned) msg.extend(serder.raw) - msg.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigs)).qb64b) # attach cnt + msg.extend(Counter(Codens.ControllerIdxSigs, count=len(sigs), + gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt for sig in sigs: msg.extend(sig.qb64b) # attach sig if couple is not None: - msg.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + msg.extend(Counter(Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(couple) return msg @@ -2094,15 +2081,16 @@ def processCuesIter(self, cues): """ while cues: # iteratively process each cue in cues msgs = bytearray() - cue = cues.pull() #cues.popleft() + cue = cues.pull() # cues.popleft() cueKin = cue["kin"] # type or kind of cue if cueKin in ("receipt",): # cue to receipt a received event from other pre cuedSerder = cue["serder"] # Serder of received event for other pre cuedKed = cuedSerder.ked cuedPrefixer = coring.Prefixer(qb64=cuedKed["i"]) - logger.info("%s got cue: kin=%s\n%s\n\n", self.pre, cueKin, - json.dumps(cuedKed, indent=1)) + logger.info("%s got cue: kin=%s%s", self.pre, cueKin, + cuedSerder.said) + logger.debug(f"event=\n{cuedSerder.pretty()}\n") if cuedKed["t"] == coring.Ilks.icp: dgkey = dbing.dgKey(self.pre, self.iserder.said) @@ -2144,6 +2132,7 @@ def processCuesIter(self, cues): # ToDo XXXX cue for kin = "psUnescrow" # ToDo XXXX cue for kin = "stream" # ToDo XXXX cue for kin = "invalid" + # ToDo XXXX cue for kin=""remoteMemberedSig"" def witnesser(self): @@ -2226,10 +2215,10 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= specified else compute default based on number of wits (backers) wits (list | None): qb64 prefixes of witnesses if any delpre (str | None): qb64 of delegator identifier prefix if any - estOnly (bool | None): True means add trait eventing.TraitCodex.EstOnly + estOnly (bool | None): True means add trait eventing.TraitDex.EstOnly which means only establishment events allowed in KEL for this Hab False (default) means allows non-est events and no trait is added. - DnD (bool): True means add trait of eventing.TraitCodex.DnD which + DnD (bool): True means add trait of eventing.TraitDex.DnD which means do not allow delegated identifiers from this identifier False (default) means do allow and no trait is added. @@ -2293,8 +2282,11 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode= self.mgr.move(old=opre, new=self.pre) # move to incept event pre # may want db method that updates .habs. and .prefixes together - habord = basing.HabitatRecord(hid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns) + # must add self.pre to self.prefixes before calling processEvent so that + # Kever.locallyOwned or Kever.locallyDelegated or Kever.locallyWitnessed + # evaluates correctly when processing own inception event. if not hidden: self.save(habord) self.prefixes.add(self.pre) @@ -2381,7 +2373,7 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, sid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True @@ -2530,8 +2522,11 @@ def replyEndRole(self, cid, role=None, eids=None, scheme=""): class SignifyGroupHab(SignifyHab): - def __init__(self, mhab=None, **kwa): + def __init__(self, smids, mhab=None, rmids=None, **kwa): self.mhab = mhab + self.smids = smids # group signing member aids in this group hab + self.rmids = rmids or smids # group rotating member aids in this group hab + super(SignifyGroupHab, self).__init__(**kwa) def make(self, *, serder, sigers, **kwargs): @@ -2540,7 +2535,8 @@ def make(self, *, serder, sigers, **kwargs): self.processEvent(serder, sigers) - habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, sid=self.pre) + habord = basing.HabitatRecord(hid=self.pre, mid=self.mhab.pre, smids=self.smids, rmids=self.rmids, + sid=self.pre, name=self.name, domain=self.ns) self.save(habord) self.inited = True @@ -2566,6 +2562,20 @@ def processEvent(self, serder, sigers): raise kering.ValidationError(f"Improper Habitat event type={serder.ked['t']} for " f"pre={self.pre}.") + def rotate(self, *, smids=None, rmids=None, serder=None, sigers=None, **kwargs): + + if (habord := self.db.habs.get(keys=(self.pre,))) is None: + raise kering.ValidationError(f"Missing HabitatRecord for pre={self.pre}") + + super(SignifyGroupHab, self).rotate(serder=serder, sigers=sigers, **kwargs) + + self.smids = smids + self.rmids = rmids + habord.smids = smids + habord.rmids = rmids + + self.db.habs.pin(keys=(self.pre,), val=habord) + class GroupHab(BaseHab): """ @@ -2647,7 +2657,7 @@ def __init__(self, smids, mhab=None, rmids=None, **kwa): """ self.mhab = mhab # local participant Hab of this group hab self.smids = smids # group signing member aids in this group hab - self.rmids = rmids # group rotating member aids in this group hab + self.rmids = rmids or smids # group rotating member aids in this group hab super(GroupHab, self).__init__(**kwa) @@ -2672,10 +2682,10 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, specified else compute default based on number of wits (backers) wits (list | None): qb64 prefixes of witnesses if any delpre (str | None): qb64 of delegator identifier prefix if any - estOnly (bool | None): True means add trait eventing.TraitCodex.EstOnly + estOnly (bool | None): True means add trait eventing.TraitDex.EstOnly which means only establishment events allowed in KEL for this Hab False (default) means allows non-est events and no trait is added. - DnD (bool): True means add trait of eventing.TraitCodex.DnD which + DnD (bool): True means add trait of eventing.TraitDex.DnD which means do not allow delegated identifiers from this identifier False (default) means do allow and no trait is added. merfers (list[Verfer] | None): group member Verfer instances of @@ -2716,6 +2726,15 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, # sign handles group hab with .mhab case sigers = self.sign(ser=serder.raw, verfers=verfers) + habord = basing.HabitatRecord(hid=self.pre, + mid=self.mhab.pre, + name=self.name, + domain=self.ns, + smids=self.smids, + rmids=self.rmids) + self.save(habord) + self.prefixes.add(self.pre) + # during delegation initialization of a habitat we ignore the MissingDelegationError and # MissingSignatureError try: @@ -2726,20 +2745,13 @@ def make(self, *, code=coring.MtrDex.Blake3_256, transferable=True, isith=None, raise kering.ConfigurationError("Improper Habitat inception for " "pre={} {}".format(self.pre, ex)) - habord = basing.HabitatRecord(hid=self.pre, - mid=self.mhab.pre, - smids=self.smids, - rmids=self.rmids) - - self.save(habord) - self.prefixes.add(self.pre) self.inited = True - def rotate(self, serder=None, smids=None, rmids=None, **kwargs): + def rotate(self, smids=None, rmids=None, serder=None, **kwargs): - if (habord := self.db.habs.get(keys=(self.name,))) is None: - raise kering.ValidationError(f"Missing HabitatRecord for name={self.name}") + if (habord := self.db.habs.get(keys=(self.pre,))) is None: + raise kering.ValidationError(f"Missing HabitatRecord for pre={self.pre}") if serder is None: msg = super(GroupHab, self).rotate(**kwargs) @@ -2763,7 +2775,7 @@ def rotate(self, serder=None, smids=None, rmids=None, **kwargs): self.rmids = rmids habord.smids = smids habord.rmids = rmids - self.db.habs.pin(keys=self.name, val=habord) + self.db.habs.pin(keys=(self.pre,), val=habord) return msg @@ -2880,6 +2892,7 @@ def query(self, pre, src, query=None, **kwa): return self.mhab.endorse(serder, last=True) + def witnesser(self): """This method name does not match logic??? """ @@ -2889,7 +2902,7 @@ def witnesser(self): if not sigs: # otherwise its a list of sigs return False - sigers = [coring.Siger(qb64b=bytes(sig)) for sig in sigs] + sigers = [indexing.Siger(qb64b=bytes(sig)) for sig in sigs] windex = min([siger.index for siger in sigers]) # True if Elected to perform delegation and witnessing diff --git a/src/keri/app/httping.py b/src/keri/app/httping.py index 6f815d4d8..f2f484c2b 100644 --- a/src/keri/app/httping.py +++ b/src/keri/app/httping.py @@ -7,7 +7,7 @@ import json from dataclasses import dataclass from urllib import parse -from urllib.parse import urlparse +from pathlib import Path import falcon from hio.base import doing @@ -41,7 +41,6 @@ def process_request(self, req, resp): req: Http request object resp: Http response object - """ sig = req.headers.get("SIGNATURE") ked = req.media @@ -152,7 +151,7 @@ def createCESRRequest(msg, client, dest, path=None): ) -def streamCESRRequests(client, ims, dest, path=None): +def streamCESRRequests(client, ims, dest, path=None, headers=None): """ Turns a stream of KERI messages into CESR http requests against the provided hio http Client @@ -167,8 +166,9 @@ def streamCESRRequests(client, ims, dest, path=None): """ path = path if path is not None else "/" + path = str(Path(client.requester.path) / path) - cold = parsing.Parser.sniff(ims) # check for spurious counters at front of stream + cold = kering.sniff(ims) # check for spurious counters at front of stream if cold in (parsing.Colds.txt, parsing.Colds.bny): # not message error out to flush stream # replace with pipelining here once CESR message format supported. raise kering.ColdStartError("Expecting message counter tritet={}" @@ -191,17 +191,19 @@ def streamCESRRequests(client, ims, dest, path=None): body = serder.raw - headers = Hict([ + headers = headers if headers is not None else Hict() + heads = (Hict([ ("Content-Type", CESR_CONTENT_TYPE), ("Content-Length", len(body)), (CESR_ATTACHMENT_HEADER, attachment), (CESR_DESTINATION_HEADER, dest) - ]) + ])) + heads.update(headers) client.request( method="POST", path=path, - headers=headers, + headers=heads, body=body ) cnt += 1 @@ -230,6 +232,9 @@ def request(self, method, url, body=None, headers=None): print(f"error establishing client connection={e}") return None + if hasattr(body, "encode"): + body = body.encode("utf-8") + client.request( method=method, path=f"{purl.path}?{purl.query}", @@ -254,7 +259,7 @@ def remove(self, client): (_, doer, _) = tup super(Clienter, self).remove([doer]) - def clientDo(self, tymth, tock=0.0): + def clientDo(self, tymth, tock=0.0, **kwa): """ Periodically prune stale clients Process existing clients and prune any that have receieved a response longer than timeout diff --git a/src/keri/app/indirecting.py b/src/keri/app/indirecting.py index 190bd23e4..7a6c709a8 100644 --- a/src/keri/app/indirecting.py +++ b/src/keri/app/indirecting.py @@ -19,10 +19,11 @@ from hio.help import decking import keri.app.oobiing -from . import directing, storing, httping, forwarding, agenting, oobiing +from . import directing, storing, httping, forwarding, agenting, oobiing, tocking from .habbing import GroupHab from .. import help, kering -from ..core import eventing, parsing, routing, coring, serdering +from ..core import (eventing, parsing, routing, coring, serdering, + Counter, Codens) from ..core.coring import Ilks from ..db import basing, dbing from ..end import ending @@ -30,6 +31,7 @@ from ..peer import exchanging from ..vdr import verifying, viring from ..vdr.eventing import Tevery +from ..metric import EscrowEnd logger = help.ogler.getLogger() @@ -40,6 +42,7 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo Setup witness controller and doers """ + host = "0.0.0.0" cues = decking.Deck() doers = [] @@ -86,16 +89,18 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo app.add_route("/", httpEnd) receiptEnd = ReceiptEnd(hab=hab, inbound=cues, aids=aids) app.add_route("/receipts", receiptEnd) - queryEnd = QueryEnd(hab=hab) + queryEnd = QueryEnd(hab=hab, reger=reger) app.add_route("/query", queryEnd) + metricsEnd = EscrowEnd(hby=hby, reger=reger) + app.add_route("/metrics", metricsEnd) - server = createHttpServer(httpPort, app, keypath, certpath, cafilepath) + server = createHttpServer(host, httpPort, app, keypath, certpath, cafilepath) if not server.reopen(): raise RuntimeError(f"cannot create http server on port {httpPort}") httpServerDoer = http.ServerDoer(server=server) # setup doers - regDoer = basing.BaserDoer(baser=verfer.reger) + regDoer = basing.BaserDoer(baser=reger) if tcpPort is not None: server = serving.Server(host="", port=tcpPort) @@ -114,12 +119,13 @@ def setupWitness(hby, alias="witness", mbx=None, aids=None, tcpPort=5631, httpPo return doers -def createHttpServer(port, app, keypath=None, certpath=None, cafilepath=None): +def createHttpServer(host, port, app, keypath=None, certpath=None, cafilepath=None): """ Create an HTTP or HTTPS server depending on whether TLS key material is present Parameters: + host(str) : host to bind to for this server, or None for default of '0.0.0.0', all ifaces port (int) : port to listen on for all HTTP(s) server instances - app (falcon.App) : application instance to pass to the http.Server instance + app (Any) : WSGI application instance to pass to the http.Server instance keypath (string) : the file path to the TLS private key certpath (string) : the file path to the TLS signed certificate (public key) cafilepath (string): the file path to the TLS CA certificate chain file @@ -132,9 +138,9 @@ def createHttpServer(port, app, keypath=None, certpath=None, cafilepath=None): certpath=certpath, cafilepath=cafilepath, port=port) - server = http.Server(port=port, app=app, servant=servant) + server = http.Server(host=host, port=port, app=app, servant=servant) else: - server = http.Server(port=port, app=app) + server = http.Server(host=host, port=port, app=app) return server @@ -158,7 +164,7 @@ def __init__(self, hab, parser, kvy, tvy, rvy, exc, cues=None, replies=None, res doers = [doing.doify(self.start), doing.doify(self.msgDo), doing.doify(self.escrowDo), doing.doify(self.cueDo)] super().__init__(doers=doers, **opts) - def start(self, tymth=None, tock=0.0): + def start(self, tymth=None, tock=0.0, **kwa): """ Prints witness name and prefix Parameters: @@ -176,7 +182,7 @@ def start(self, tymth=None, tock=0.0): print("Witness", self.hab.name, ":", self.hab.pre) - def msgDo(self, tymth=None, tock=0.0): + def msgDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -190,15 +196,15 @@ def msgDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessMsgTock _ = (yield self.tock) if self.parser.ims: - logger.info("Client %s received:\n%s\n...\n", self.kvy, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + logger.debug("Client %s received:\n%s\n...\n", self.kvy, self.parser.ims[:1024]) + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def escrowDo(self, tymth=None, tock=0.0): + def escrowDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery and .tevery escrows. @@ -212,7 +218,7 @@ def escrowDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessEscrowTock _ = (yield self.tock) while True: @@ -222,9 +228,9 @@ def escrowDo(self, tymth=None, tock=0.0): self.tvy.processEscrows() self.exc.processEscrow() - yield + yield self.tock - def cueDo(self, tymth=None, tock=0.0): + def cueDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -243,12 +249,12 @@ def cueDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.WitnessCueTock _ = (yield self.tock) while True: while self.cues: - cue = self.cues.pull() # self.cues.popleft() + cue = self.cues.pull() # self.cues.popleft() cueKin = cue["kin"] if cueKin == "stream": self.queries.append(cue) @@ -360,7 +366,7 @@ def wind(self, tymth): super(Indirector, self).wind(tymth) self.client.wind(tymth) - def msgDo(self, tymth=None, tock=0.0): + def msgDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -379,15 +385,15 @@ def msgDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.IndirectorMsgTock _ = (yield self.tock) if self.parser.ims: - logger.info("Client %s received:\n%s\n...\n", self.hab.pre, self.parser.ims[:1024]) - done = yield from self.parser.parsator() # process messages continuously + logger.debug("Client %s received:\n%s\n...\n", self.hab.pre, self.parser.ims[:1024]) + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def cueDo(self, tymth=None, tock=0.0): + def cueDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery.cues deque @@ -406,16 +412,16 @@ def cueDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.IndirectorCueTock _ = (yield self.tock) while True: for msg in self.hab.processCuesIter(self.kevery.cues): self.sendMessage(msg, label="chit or receipt") - yield # throttle just do one cue at a time - yield + yield self.tock # throttle just do one cue at a time + yield self.tock - def escrowDo(self, tymth=None, tock=0.0): + def escrowDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -434,19 +440,19 @@ def escrowDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.IndirectorEscrowTock _ = (yield self.tock) while True: self.kevery.processEscrows() - yield + yield self.tock def sendMessage(self, msg, label=""): """ Sends message msg and loggers label if any """ self.client.tx(msg) # send to remote - logger.info("%s sent %s:\n%s\n\n", self.hab.pre, label, bytes(msg)) + logger.debug("%s sent %s:\n%s\n\n", self.hab.pre, label, bytes(msg)) class MailboxDirector(doing.DoDoer): @@ -476,13 +482,11 @@ class MailboxDirector(doing.DoDoer): hby (Habitat: local controller's context Properties: - tyme (float): relative cycle time of associated Tymist, obtained - via injected .tymth function wrapper closure. - tymth (function): function wrapper closure returned by Tymist .tymeth() - method. When .tymth is called it returns associated Tymist .tyme. - .tymth provides injected dependency on Tymist tyme base. - tock (float): desired time in seconds between runs or until next run, - non negative, zero means run asap + hby (Habery): the Habery in which mailbox messages are routed + verifier (Verifier): TEL event acceptor and validator + exchanger (Exchanger): Exchange (exn) message delivery component + rep (Respondant): Respondant for reply messages + cues (Deck): Queue for new actions to schedule shared between the Revery, Kevery (and Kever), and Tevery (and Tever) Methods: @@ -506,7 +510,7 @@ class MailboxDirector(doing.DoDoer): """ def __init__(self, hby, topics, ims=None, verifier=None, kvy=None, exc=None, rep=None, cues=None, rvy=None, - tvy=None, **kwa): + tvy=None, witnesses=True, **kwa): """ Initialize instance. @@ -530,6 +534,7 @@ def __init__(self, hby, topics, ims=None, verifier=None, kvy=None, exc=None, rep self.pollers = list() self.prefixes = oset() self.cues = cues if cues is not None else decking.Deck() + self.witnesses = witnesses self.ims = ims if ims is not None else bytearray() @@ -576,7 +581,7 @@ def wind(self, tymth): """ super(MailboxDirector, self).wind(tymth) - def pollDo(self, tymth=None, tock=0.0): + def pollDo(self, tymth=None, tock=None, **kwa): """ Returns: doifiable Doist compatible generator method @@ -586,7 +591,7 @@ def pollDo(self, tymth=None, tock=0.0): """ # enter context self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.MailboxPollTock _ = (yield self.tock) habs = list(self.hby.habs.values()) @@ -622,11 +627,12 @@ def addPollers(self, hab): self.pollers.append(poller) self.extend([poller]) - wits = hab.kever.wits - for wit in wits: - poller = Poller(hab=hab, topics=self.topics, witness=wit) - self.pollers.append(poller) - self.extend([poller]) + if self.witnesses: + wits = hab.kever.wits + for wit in wits: + poller = Poller(hab=hab, topics=self.topics, witness=wit) + self.pollers.append(poller) + self.extend([poller]) self.prefixes.add(hab.pre) @@ -653,7 +659,7 @@ def processPollIter(self): msg = mail.pop(0) yield msg - def msgDo(self, tymth=None, tock=0.0): + def msgDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process incoming message stream of .kevery @@ -672,13 +678,13 @@ def msgDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.MailboxMsgTock _ = (yield self.tock) - done = yield from self.parser.parsator() # process messages continuously + done = yield from self.parser.parsator(local=True) # process messages continuously return done # should nover get here except forced close - def escrowDo(self, tymth=None, tock=0.0): + def escrowDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .kevery escrows. @@ -697,7 +703,7 @@ def escrowDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.MailboxEscrowTock _ = (yield self.tock) while True: @@ -710,7 +716,7 @@ def escrowDo(self, tymth=None, tock=0.0): if self.verifier is not None: self.verifier.processEscrows() - yield + yield self.tock @property def times(self): @@ -751,7 +757,7 @@ def __init__(self, hab, witness, topics, msgs=None, retry=1000, **kwa): super(Poller, self).__init__(doers=doers, **kwa) - def eventDo(self, tymth=None, tock=0.0): + def eventDo(self, tymth=None, tock=None, **kwa): """ Returns: doifiable Doist compatible generator method @@ -760,7 +766,7 @@ def eventDo(self, tymth=None, tock=0.0): add result of doify on this method to doers list """ self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.PollerEventTock _ = (yield self.tock) witrec = self.hab.db.tops.get((self.pre, self.witness)) @@ -891,7 +897,7 @@ def on_post(self, req, rep): rep.set_header('connection', "close") cr = httping.parseCesrHttpRequest(req=req) - sadder = coring.Sadder(ked=cr.payload, kind=eventing.Serials.json) + sadder = coring.Sadder(ked=cr.payload, kind=eventing.Kinds.json) msg = bytearray(sadder.raw) msg.extend(cr.attachments.encode("utf-8")) @@ -971,7 +977,7 @@ def __iter__(self): def __next__(self): if self.iter is None: if self.cues: - cue = self.cues.pull() # self.cues.popleft() + cue = self.cues.pull() serder = cue["serder"] if serder.said == self.said: kin = cue["kin"] @@ -1061,7 +1067,7 @@ def on_post(self, req, rep): rep.set_header('connection', "close") cr = httping.parseCesrHttpRequest(req=req) - serder = serdering.SerderKERI(sad=cr.payload, kind=eventing.Serials.json) + serder = serdering.SerderKERI(sad=cr.payload, kind=eventing.Kinds.json) pre = serder.ked["i"] if self.aids is not None and pre not in self.aids: @@ -1074,7 +1080,7 @@ def on_post(self, req, rep): msg = bytearray(serder.raw) msg.extend(cr.attachments.encode("utf-8")) - self.psr.parseOne(ims=msg) + self.psr.parseOne(ims=msg, local=True) if pre in self.hab.kevers: kever = self.hab.kevers[pre] @@ -1140,8 +1146,8 @@ def on_get(self, req, rep): said=said.decode("utf-8")) rct = bytearray(rserder.raw) if wigs := self.hab.db.getWigs(key=dgkey): - rct.extend(coring.Counter(code=coring.CtrDex.WitnessIdxSigs, - count=len(wigs)).qb64b) + rct.extend(Counter(Codens.WitnessIdxSigs, count=len(wigs), + gvrsn=kering.Vrsn_1_0).qb64b) for wig in wigs: rct.extend(wig) @@ -1149,7 +1155,7 @@ def on_get(self, req, rep): rep.status = falcon.HTTP_200 rep.data = rct - def interceptDo(self, tymth=None, tock=0.0): + def interceptDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process Kevery and Tevery cues deque @@ -1159,7 +1165,7 @@ def interceptDo(self, tymth=None, tock=0.0): """ # enter context self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.ReceiptInterceptTock _ = (yield self.tock) while True: @@ -1187,9 +1193,9 @@ class QueryEnd: """ - def __init__(self, hab): + def __init__(self, hab, reger): self.hab = hab - self.reger = viring.Reger(name=hab.name, db=hab.db, temp=False) + self.reger = reger def on_get(self, req, rep): """ Handles GET requests to query KEL or TEL events of a pre from a witness. @@ -1280,4 +1286,4 @@ def on_get(self, req, rep): else: rep.set_header('Content-Type', "application/json") rep.text = "unkown query type." - rep.status = falcon.HTTP_400 \ No newline at end of file + rep.status = falcon.HTTP_400 diff --git a/src/keri/app/keeping.py b/src/keri/app/keeping.py index 085957fa5..16ab4931b 100644 --- a/src/keri/app/keeping.py +++ b/src/keri/app/keeping.py @@ -23,6 +23,7 @@ """ import math +import os from collections import namedtuple, deque from dataclasses import dataclass, asdict, field @@ -30,10 +31,13 @@ from hio.base import doing from .. import kering +from .. import core, help from ..core import coring from ..db import dbing, subing, koming from ..help import helping +logger = help.ogler.getLogger() + Algoage = namedtuple("Algoage", 'randy salty group extern') Algos = Algoage(randy='randy', salty='salty', group="group", extern="extern") # randy is rerandomize, salty is use salt @@ -128,6 +132,9 @@ def openKS(name="test", **kwa): """ return dbing.openLMDB(cls=Keeper, name=name, **kwa) +# Env var for configuring LMDB size for the Keeper database +KERIKeeperMapSizeKey = "KERI_KEEPER_MAP_SIZE" + class Keeper(dbing.LMDBer): """ @@ -250,6 +257,15 @@ def __init__(self, headDirPath=None, perm=None, reopen=False, **kwa): if perm is None: perm = self.Perm # defaults to restricted permissions for non temp + mapSize = os.getenv(dbing.KERIKeeperMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIKeeperMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Keeper, self).__init__(headDirPath=headDirPath, perm=perm, reopen=reopen, **kwa) @@ -267,10 +283,10 @@ def reopen(self, **kwa): self.pris = subing.CryptSignerSuber(db=self, subkey='pris.') self.prxs = subing.CesrSuber(db=self, subkey='prxs.', - klas=coring.Cipher) + klas=core.Cipher) self.nxts = subing.CesrSuber(db=self, subkey='nxts.', - klas=coring.Cipher) + klas=core.Cipher) self.smids = subing.CatCesrIoSetSuber(db=self, subkey='smids.', klas=(coring.Prefixer, coring.Seqner)) @@ -342,7 +358,7 @@ def __init__(self, keeper, **kwa): self.keeper = keeper - def enter(self): + def enter(self, *, temp=False): """""" if not self.keeper.opened: self.keeper.reopen() @@ -444,7 +460,7 @@ def create(self, codes=None, count=1, code=coring.MtrDex.Ed25519_Seed, codes = [code for i in range(count)] for code in codes: - signers.append(coring.Signer(code=code, transferable=transferable)) + signers.append(core.Signer(code=code, transferable=transferable)) return signers @@ -477,7 +493,7 @@ def __init__(self, salt=None, stem=None, tier=None, **kwa): """ super(SaltyCreator, self).__init__(**kwa) - self.salter = coring.Salter(qb64=salt, tier=tier) + self.salter = core.Salter(qb64=salt, tier=tier) self._stem = stem if stem is not None else '' @property @@ -596,9 +612,9 @@ class Manager: Attributes: ks (Keeper): key store LMDB database instance for storing public and private keys - encrypter (coring.Encrypter): instance for encrypting secrets. Public + encrypter (core.Encrypter): instance for encrypting secrets. Public encryption key is derived from aeid (public signing key) - decrypter (coring.Decrypter): instance for decrypting secrets. Private + decrypter (core.Decrypter): instance for decrypting secrets. Private decryption key is derived seed (private signing key seed) inited (bool): True means fully initialized wrt database. False means not yet fully initialized @@ -724,13 +740,13 @@ def setup(self, aeid=None, pidx=None, algo=None, salt=None, tier=None): if algo is None: algo = Algos.salty if salt is None: - salt = coring.Salter().qb64 + salt = core.Salter().qb64 else: - if coring.Salter(qb64=salt).qb64 != salt: + if core.Salter(qb64=salt).qb64 != salt: raise ValueError(f"Invalid qb64 for salt={salt}.") if tier is None: - tier = coring.Tiers.low + tier = core.Tiers.low # update database if never before initialized if self.pidx is None: # never before initialized @@ -749,13 +765,13 @@ def setup(self, aeid=None, pidx=None, algo=None, salt=None, tier=None): if not self.aeid: # never before initialized self.updateAeid(aeid, self.seed) else: - self.encrypter = coring.Encrypter(verkey=self.aeid) # derive encrypter from aeid + self.encrypter = core.Encrypter(verkey=self.aeid) # derive encrypter from aeid if not self.seed or not self.encrypter.verifySeed(self.seed): raise kering.AuthError("Last seed missing or provided last seed " "not associated with last aeid={}." "".format(self.aeid)) - self.decrypter = coring.Decrypter(seed=self.seed) + self.decrypter = core.Decrypter(seed=self.seed) self.inited = True @@ -781,7 +797,7 @@ def updateAeid(self, aeid, seed): if aeid: # aeid provided if aeid != self.aeid: # changing to a new aeid so update .encrypter - self.encrypter = coring.Encrypter(verkey=aeid) # derive encrypter from aeid + self.encrypter = core.Encrypter(verkey=aeid) # derive encrypter from aeid # verifies new seed belongs to new aeid if not seed or not self.encrypter.verifySeed(seed): raise kering.AuthError("Seed missing or provided seed not associated" @@ -803,8 +819,8 @@ def updateAeid(self, aeid, seed): # re-encrypt root salt secrets by prefix parameters .prms for keys, data in self.ks.prms.getItemIter(): # keys is tuple of pre qb64 if data.salt: - salter = self.decrypter.decrypt(ser=data.salt) - data.salt = (self.encrypter.encrypt(matter=salter).qb64 + salter = self.decrypter.decrypt(qb64=data.salt) + data.salt = (self.encrypter.encrypt(prim=salter).qb64 if self.encrypter else salter.qb64) self.ks.prms.pin(keys, val=data) @@ -817,7 +833,7 @@ def updateAeid(self, aeid, seed): self._seed = seed # set .seed in memory # update .decrypter - self.decrypter = coring.Decrypter(seed=seed) if seed else None + self.decrypter = core.Decrypter(seed=seed) if seed else None @property @@ -888,7 +904,7 @@ def salt(self): """ salt = self.ks.gbls.get('salt') if self.decrypter: # given .decrypt secret salt must be encrypted in db - return self.decrypter.decrypt(ser=salt).qb64 + return self.decrypter.decrypt(qb64=salt).qb64 return salt @@ -901,7 +917,7 @@ def salt(self, salt): may be plain text or cipher text handled by updateAeid """ if self.encrypter: - salt = self.encrypter.encrypt(ser=salt).qb64 + salt = self.encrypter.encrypt(ser=salt, code=core.MtrDex.X25519_Cipher_Salt).qb64 self.ks.gbls.pin('salt', salt) @@ -1019,7 +1035,8 @@ def incept(self, icodes=None, icount=1, icode=coring.MtrDex.Ed25519_Seed, if creator.salt: pp.salt = (creator.salt if not self.encrypter - else self.encrypter.encrypt(ser=creator.salt).qb64) + else self.encrypter.encrypt(ser=creator.salt, + code=core.MtrDex.X25519_Cipher_Salt).qb64) dt = helping.nowIso8601() ps = PreSit( @@ -1183,9 +1200,9 @@ def rotate(self, pre, ncodes=None, ncount=1, if self.aeid: if not self.decrypter: raise kering.DecryptError("Unauthorized decryption. Aeid but no decrypter.") - salt = self.decrypter.decrypt(ser=salt).qb64 + salt = self.decrypter.decrypt(qb64=salt).qb64 else: - salt = coring.Salter(qb64=salt).qb64 # ensures salt was unencrypted + salt = core.Salter(qb64=salt).qb64 # ensures salt was unencrypted creator = Creatory(algo=pp.algo).make(salt=salt, stem=pp.stem, tier=pp.tier) @@ -1393,21 +1410,23 @@ def sign(self, ser, pubs=None, verfers=None, indexed=True, cigars.append(signer.sign(ser)) # assigns .verfer to cigar return cigars - def decrypt(self, ser, pubs=None, verfers=None): + + def decrypt(self, qb64, pubs=None, verfers=None): """ - Returns list of signatures of ser if indexed as Sigers else as Cigars with - .verfer assigned. + Returns decrypted plaintext of encrypted qb64 ciphertext serialization. Parameters: - ser (bytes): serialization to sign + qb64 (str | bytes | bytearray | memoryview): fully qualified base64 + ciphertext serialization to decrypt pubs (list[str] | None): of qb64 public keys to lookup private keys one of pubs or verfers is required. If both then verfers is ignored. verfers (list[Verfer] | None): Verfer instances of public keys one of pubs or verfers is required. If both then verfers is ignored. - If not pubs then gets public key from verfer.qb64 + If not pubs then gets public key from verfer.qb64 used to lookup + private keys Returns: - bytes: decrypted data + plain (bytes): decrypted plaintext """ signers = [] @@ -1432,18 +1451,22 @@ def decrypt(self, ser, pubs=None, verfers=None): raise ValueError("Missing prikey in db for pubkey={}".format(verfer.qb64)) signers.append(signer) - plain = ser + if hasattr(qb64, "encode"): + qb64 = qb64.encode() # convert str to bytes + qb64 = bytes(qb64) # convert bytearray or memoryview to bytes + for signer in signers: sigkey = signer.raw + signer.verfer.raw # sigkey is raw seed + raw verkey prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) # raw private encrypt key pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) - plain = pysodium.crypto_box_seal_open(plain, pubkey, prikey) # qb64b + plain = pysodium.crypto_box_seal_open(qb64, pubkey, prikey) # qb64b - if plain == ser: - raise ValueError("unable to decrypt data") + if plain == qb64: + raise ValueError(f"Unable to decrypt.") return plain + def ingest(self, secrecies, iridx=0, ncount=1, ncode=coring.MtrDex.Ed25519_Seed, dcode=coring.MtrDex.Blake3_256, algo=Algos.salty, salt=None, stem=None, tier=None, @@ -1531,7 +1554,7 @@ def ingest(self, secrecies, iridx=0, ncount=1, ncode=coring.MtrDex.Ed25519_Seed, secrecies = deque(secrecies) while secrecies: csecrets = secrecies.popleft() # current - csigners = [coring.Signer(qb64=secret, transferable=transferable) + csigners = [core.Signer(qb64=secret, transferable=transferable) for secret in csecrets] csize = len(csigners) verferies.append([signer.verfer for signer in csigners]) @@ -1541,7 +1564,8 @@ def ingest(self, secrecies, iridx=0, ncount=1, ncode=coring.MtrDex.Ed25519_Seed, pp = PrePrm(pidx=pidx, algo=algo, salt=(creator.salt if not self.encrypter - else self.encrypter.encrypt(ser=creator.salt).qb64), + else self.encrypter.encrypt(ser=creator.salt, + code=core.MtrDex.X25519_Cipher_Salt).qb64), stem=creator.stem, tier=creator.tier) pre = csigners[0].verfer.qb64b @@ -1747,7 +1771,7 @@ def __init__(self, manager, **kwa): self.manager = manager - def enter(self): + def enter(self, *, temp=False): """""" if not self.manager.inited: self.manager.setup(**self.manager._inits) diff --git a/src/keri/app/notifying.py b/src/keri/app/notifying.py index a48adc61b..128c115a4 100644 --- a/src/keri/app/notifying.py +++ b/src/keri/app/notifying.py @@ -3,15 +3,18 @@ keri.app.notifying module """ +import os from collections.abc import Iterable from typing import Union, Type -from keri import kering +from keri import kering, help from keri.help import helping from keri.app import signaling from keri.core import coring from keri.db import dbing, subing +logger = help.ogler.getLogger() + def notice(attrs, dt=None, read=False): """ @@ -180,11 +183,12 @@ def getItemIter(self, keys: Union[str, Iterable] = b""): keys (tuple): of key strs to be combined in order to form key Returns: - iterator: of tuples of keys tuple and val coring.Serder for + iterator: of tuples of keys tuple and val serdering.SerderKERI for each entry in db """ - for key, val in self.db.getAllItemIter(db=self.sdb, key=self._tokey(keys), split=False): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys)): yield self._tokeys(key), self.klas(raw=bytes(val)) def cntAll(self): @@ -196,6 +200,9 @@ def cntAll(self): """ return self.db.cnt(db=self.sdb) +# Env var for configuring LMDB size for the Noter database +KERINoterMapSizeKey = "KERI_NOTER_MAP_SIZE" + class Noter(dbing.LMDBer): """ @@ -220,6 +227,15 @@ def __init__(self, name="not", headDirPath=None, reopen=True, **kwa): self.nidx = None self.ncigs = None + mapSize = os.getenv(dbing.KERINoterMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERINoterMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Noter, self).__init__(name=name, headDirPath=headDirPath, reopen=reopen, **kwa) def reopen(self, **kwa): diff --git a/src/keri/app/oobiing.py b/src/keri/app/oobiing.py index 3a45c122b..8b5e8419c 100644 --- a/src/keri/app/oobiing.py +++ b/src/keri/app/oobiing.py @@ -13,18 +13,18 @@ import falcon from hio.base import doing from hio.help import decking -from keri.core import coring +from keri.core import coring from . import httping -from .habbing import GroupHab from .. import help from .. import kering -from ..app import forwarding, connecting +from ..app import connecting from ..core import routing, eventing, parsing, scheming, serdering from ..db import basing from ..end import ending from ..end.ending import OOBI_RE, DOOBI_RE from ..help import helping +from ..kering import Ilks, ValidationError, UnverifiedReplyError, ConfigurationError from ..peer import exchanging logger = help.ogler.getLogger() @@ -115,15 +115,15 @@ def on_get_alias(self, req, rep, alias=None): if role in (kering.Roles.witness,): # Fetch URL OOBIs for all witnesses oobis = [] for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) \ + or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: rep.status = falcon.HTTP_404 rep.text = f"unable to query witness {wit}, no http endpoint" return - url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness/{wit}") + url = urls[kering.Schemes.https] if kering.Schemes.https in urls else urls[kering.Schemes.http] + oobis.append(f"{url.rstrip("/")}/oobi/{hab.pre}/witness/{wit}") res["oobis"] = oobis elif role in (kering.Roles.controller,): # Fetch any controller URL OOBIs oobis = [] @@ -134,8 +134,7 @@ def on_get_alias(self, req, rep, alias=None): rep.text = f"unable to query controller {hab.pre}, no http endpoint" return url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(url) - oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + oobis.append(f"{url.rstrip("/")}/oobi/{hab.pre}/controller") res["oobis"] = oobis else: rep.status = falcon.HTTP_404 @@ -276,7 +275,7 @@ class Oobiery: RetryDelay = 30 - def __init__(self, hby, clienter=None, cues=None): + def __init__(self, hby, rvy=None, clienter=None, cues=None): """ DoDoer to handle the request and parsing of OOBIs Parameters: @@ -286,8 +285,14 @@ def __init__(self, hby, clienter=None, cues=None): """ self.hby = hby + self.rvy = rvy + if self.rvy is not None: + self.registerReplyRoutes(self.rvy.rtr) + self.clienter = clienter or httping.Clienter() self.org = connecting.Organizer(hby=self.hby) + + # Set up a local parser for returned events from OOBI queries. rtr = routing.Router() rvy = routing.Revery(db=self.hby.db, rtr=rtr) kvy = eventing.Kevery(db=self.hby.db, lax=True, local=False, rvy=rvy) @@ -298,7 +303,90 @@ def __init__(self, hby, clienter=None, cues=None): self.clients = dict() self.doers = [self.clienter, doing.doify(self.scoobiDo)] - def scoobiDo(self, tymth=None, tock=0.0): + def registerReplyRoutes(self, router): + """ Register the routes for processing messages embedded in `rpy` event messages + + The Oobiery handles rpy messages with the /introduce route by processing the contained oobi + + Parameters: + router(Router): reply message router + + """ + router.addRoute("/introduce", self) + + def processReply(self, *, serder, saider, route, cigars=None, tsgs=None, **kwargs): + """ + Process one reply message for route = /introduce + with either attached nontrans receipt couples in cigars or attached trans + indexed sig groups in tsgs. + Assumes already validated saider, dater, and route from serder.ked + + Parameters: + serder (SerderKERI): instance of reply msg (SAD) + saider (Saider): instance from said in serder (SAD) + route (str): reply route + cigars (list): of Cigar instances that contain nontrans signing couple + signature in .raw and public key in .verfer + tsgs (list): tuples (quadruples) of form + (prefixer, seqner, diger, [sigers]) where: + prefixer is pre of trans endorser + seqner is sequence number of trans endorser's est evt for keys for sigs + diger is digest of trans endorser's est evt for keys for sigs + [sigers] is list of indexed sigs from trans endorser's keys from est evt + + OobiRecord: + date: str = date time of reply message of the introduction + + Reply Message: + { + "v" : "KERI10JSON00011c_", + "t" : "rpy", + "d": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "dt": "2020-08-22T17:50:12.988921+00:00", + "r" : "/introduce", + "a" : + { + "cid": "ENcOes8_t2C7tck4X4j61fSm0sWkLbZrEZffq7mSn8On", + "oobi": "http://localhost:5632/oobi/ENcOes8_t2C7tck4X4j61fSm0sWkLbZrEZffq7mSn8On/witness", + } + } + + """ + if route != "/introduce": + raise ValidationError(f"Usupported route={route} in {Ilks.rpy} " + f"msg={serder.ked}.") + + data = serder.ked['a'] + dt = serder.ked["dt"] + + for k in ("cid", "oobi"): + if k not in data: + raise ValidationError(f"Missing element={k} from attributes in" + f" {Ilks.rpy} msg={serder.ked}.") + + cider = coring.Prefixer(qb64=data["cid"]) # raises error if unsupported code + cid = cider.qb64 # controller authorizing eid at role + aid = cid # authorizing attribution id + + oobi = data["oobi"] + url = urlparse(oobi) + if url.scheme not in ("http", "https"): + raise ValidationError(f"Invalid url scheme for introduced OOBI scheme={url.scheme}") + + if self.rvy is None: + raise ConfigurationError("this oobiery is not configured to handle rpy introductions") + + # Process BADA RUN but with no previous reply message, always process introductions + accepted = self.rvy.acceptReply(serder=serder, saider=saider, route=route, + aid=aid, osaider=None, cigars=cigars, + tsgs=tsgs) + if not accepted: + raise UnverifiedReplyError(f"Unverified introduction reply. {serder.ked}") + + obr = basing.OobiRecord(cid=cid, date=dt) + self.hby.db.oobis.put(keys=(oobi,), val=obr) + + def scoobiDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .exc responses and pass them on to the HTTPRespondant @@ -573,7 +661,7 @@ def addAuthToAid(self, cid, url): wkan = basing.WellKnownAuthN(url=url, dt=now) self.hby.db.wkas.add(keys=(cid,), val=wkan) - def authzDo(self, tymth=None, tock=0.0): + def authzDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process .exc responses and pass them on to the HTTPRespondant diff --git a/src/keri/app/querying.py b/src/keri/app/querying.py index 2dc4ce17d..21ff01297 100644 --- a/src/keri/app/querying.py +++ b/src/keri/app/querying.py @@ -90,13 +90,17 @@ def recur(self, tyme, deeds=None): class SeqNoQuerier(doing.DoDoer): - def __init__(self, hby, hab, pre, sn, **opts): + def __init__(self, hby, hab, pre, sn, fn=None, wits=None, **opts): self.hby = hby self.hab = hab self.pre = pre self.sn = sn + self.fn = fn if fn is not None else 0 self.witq = agenting.WitnessInquisitor(hby=self.hby) - self.witq.query(src=self.hab.pre, pre=self.pre, sn="{:x}".format(self.sn)) + self.witq.query(src=self.hab.pre, pre=self.pre, + sn="{:x}".format(self.sn), + fn="{:x}".format(self.fn), + wits=wits) super(SeqNoQuerier, self).__init__(doers=[self.witq], **opts) def recur(self, tyme, deeds=None): @@ -136,8 +140,7 @@ def recur(self, tyme, deeds=None): if self.pre not in self.hab.kevers: return False - kever = self.hab.kevers[self.pre] - if self.hby.db.findAnchoringSealEvent(self.pre, seal=self.anchor): + if self.hby.db.fetchLastSealingEventByEventSeal(self.pre, seal=self.anchor): self.remove([self.witq]) return True diff --git a/src/keri/app/signaling.py b/src/keri/app/signaling.py index f32e9dbe2..edec6876a 100644 --- a/src/keri/app/signaling.py +++ b/src/keri/app/signaling.py @@ -27,7 +27,7 @@ def signal(attrs, topic, ckey=None, dt=None): Notice: Notice instance """ - dt = dt if dt is not None else datetime.datetime.now().isoformat() + dt = dt if dt is not None else helping.nowIso8601() if hasattr(dt, "isoformat"): dt = dt.isoformat() @@ -117,7 +117,7 @@ def push(self, attrs, topic, ckey=None, dt=None): Returns: """ - dt = dt if dt is not None else datetime.datetime.now() + dt = dt if dt is not None else helping.nowIso8601() sig = signal(attrs=attrs, topic=topic, ckey=ckey, dt=dt) if sig.ckey is not None: @@ -128,7 +128,7 @@ def push(self, attrs, topic, ckey=None, dt=None): self.signals.append(sig) - def expireDo(self, tymth=None, tock=0.0): + def expireDo(self, tymth=None, tock=0.0, **kwa): """ Returns doifiable Doist compatible generator method (doer dog) @@ -146,7 +146,7 @@ def expireDo(self, tymth=None, tock=0.0): _ = (yield self.tock) while True: # loop checking for expired messages - now = datetime.datetime.now() + now = helping.nowUTC() toRemove = [] for sig in self.signals: if now - helping.fromIso8601(sig.dt) > self.SignalTimeout: # Expire messages that are too old diff --git a/src/keri/app/signing.py b/src/keri/app/signing.py index f8aed4456..3c99c50f5 100644 --- a/src/keri/app/signing.py +++ b/src/keri/app/signing.py @@ -4,13 +4,17 @@ keri.app.signing module """ +from ..kering import Vrsn_1_0, Vrsn_2_0 from ..app.habbing import GroupHab -from ..core import coring, eventing +from .. import core +from ..core import coring, eventing, counting + def serialize(creder, prefixer, seqner, saider): craw = bytearray(creder.raw) - craw.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + craw.extend(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=Vrsn_1_0).qb64b) craw.extend(prefixer.qb64b) craw.extend(seqner.qb64b) craw.extend(saider.qb64b) diff --git a/src/keri/app/storing.py b/src/keri/app/storing.py index 99b38cf37..2a18dd0b0 100644 --- a/src/keri/app/storing.py +++ b/src/keri/app/storing.py @@ -3,12 +3,13 @@ keri.app.storing module """ +import os from hio.base import doing from hio.help import decking from ordered_set import OrderedSet as oset -from . import forwarding +from . import forwarding, tocking from .. import help from ..core import coring, serdering from ..core.coring import MtrDex @@ -16,6 +17,9 @@ logger = help.ogler.getLogger() +# Env var for configuring LMDB size for the Mailbox database +KERIMailboxerMapSizeKey = "KERI_MAILBOXER_MAP_SIZE" + class Mailboxer(dbing.LMDBer): """ @@ -34,10 +38,30 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa): perm: reopen: kwa: + + Mailboxer uses two dbs for mailbox messages these are .tpcs and .msgs. + The message index is in .tpcs (topics). + Each .tpcs index key consists of topic.on where topic is bytes + identifier or prefix/topic for message and on is serialized + ordinal number to orders the appearance of a topic message. + Eash .tpcs val is the digest of the message. + The message itself is stored in .msgs where the key is the msg digest + and the value is the serialized messag itself. + Multiple messages can share the same topic but with a different ordinal. + """ self.tpcs = None self.msgs = None + mapSize = os.getenv(dbing.KERIMailboxerMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIMailboxerMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Mailboxer, self).__init__(name=name, headDirPath=headDirPath, reopen=reopen, **kwa) def reopen(self, **kwa): @@ -47,88 +71,102 @@ def reopen(self, **kwa): :return: """ super(Mailboxer, self).reopen(**kwa) - - self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True) + self.tpcs = subing.OnSuber(db=self, subkey='tpcs.') self.msgs = subing.Suber(db=self, subkey='msgs.') # key states return self.env - def delTopic(self, key): - """ - Use snKey() - Deletes value at key. - Returns True If key exists in database Else False + def delTopic(self, key, on=0): + """Removes topic index from .tpcs without deleting message from .msgs + + Returns: + result (boo): True if full key consisting of key and serialized on + exists in database so removed + False otherwise (not removed) """ - return self.delIoSetVals(self.tpcs, key) + return self.tpcs.remOn(keys=key, on=on) def appendToTopic(self, topic, val): - """ - Return first seen order number int, fn, of appended entry. - Computes fn as next fn after last entry. + """Appends val to end of db entries with same topic but with on + incremented by 1 relative to last preexisting entry at topic. - Append val to end of db entries with same topic but with fn incremented by - 1 relative to last preexisting entry at pre. + Returns: + on (int): order number int, on, of appended entry. + Computes on as next on after last entry. Parameters: - topic is bytes identifier prefix/topic for message - val is event digest + topic (bytes): topic identifier for message + val (bytes): msg digest """ - return self.appendIoSetVal(db=self.tpcs, key=topic, val=val) + return self.tpcs.appendOn(key=topic, val=val) + def getTopicMsgs(self, topic, fn=0): """ Returns: - ioset (oset): the insertion ordered set of values at same apparent - effective key. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. + msgs (Iterable[bytes]): belonging to topic indices with same topic but all + on >= fn i.e. all topic.on beginning with fn Parameters: - topic (Option(bytes|str)): Apparent effective key - fn (int) starting index + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - - digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn) msgs = [] - for dig in digs: + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): if msg := self.msgs.get(keys=dig): - msgs.append(msg.encode("utf-8")) + msgs.append(msg.encode()) # want bytes not str return msgs + def storeMsg(self, topic, msg): """ - Add exn event to mailbox of dest identifier + Add exn event to mailbox topic and on that is 1 greater than last msg + at topic. + + Returns: + result (bool): True if msg successfully stored and indexed at topic + False otherwise Parameters: - msg (bytes): - topic (qb64b): + topic (str | bytes): topic (Option(bytes|str)): key prefix combined + with serialized on to form full actual key. + msg (bytes): serialized message """ - if hasattr(topic, "encode"): - topic = topic.encode("utf-8") - if hasattr(msg, "encode"): msg = msg.encode("utf-8") digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b - self.appendToTopic(topic=topic, val=digb) + on = self.tpcs.appendOn(keys=topic, val=digb) return self.msgs.pin(keys=digb, val=msg) + def cloneTopicIter(self, topic, fn=0): """ - Returns iterator of first seen exn messages with attachments for the - identifier prefix pre starting at first seen order number, fn. + Returns: + triple (Iterator[(on, topic, msg): iterator of messages at topic + beginning with ordinal fn. + + topic (Option(bytes|str)): key prefix combined with serialized on + to form full actual key. When key is empty then retrieves + whole database. + fn (int): starting index ordinal number used with onKey(pre,on) + to form key at at which to initiate retrieval - """ - if hasattr(topic, 'encode'): - topic = topic.encode("utf-8") - for (key, dig) in self.getIoSetItemsIter(self.tpcs, key=topic, ion=fn): - topic, ion = dbing.unsuffix(key) + + ToDo looks like misuse of IoSet this should not be IoSet but simply + Ordinal Numbered db. since should not be using hidden ion has not + hidden. + + """ + for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn): if msg := self.msgs.get(keys=dig): - yield ion, topic, msg.encode("utf-8") + yield (on, topic, msg.encode("utf-8")) + class Respondant(doing.DoDoer): @@ -161,7 +199,7 @@ def __init__(self, hby, reps=None, cues=None, mbx=None, aids=None, **kwa): doers = [self.postman, doing.doify(self.responseDo), doing.doify(self.cueDo)] super(Respondant, self).__init__(doers=doers, **kwa) - def responseDo(self, tymth=None, tock=0.0): + def responseDo(self, tymth=None, tock=0.0, **kwa): """ Doifiable Doist compatibile generator method to process response messages from `exn` handlers. If dest is not in local environment, ignore the response (for now). If dest has witnesses, @@ -199,13 +237,15 @@ def responseDo(self, tymth=None, tock=0.0): # sign the exn to get the signature eattach = senderHab.endorse(exn, last=False, pipelined=False) del eattach[:exn.size] + logger.info("Sending exn on %s from %s to %s", topic, sender, recipient) + logger.debug("exn body=\n%s\n", exn.pretty()) self.postman.send(recipient, topic=topic, serder=exn, hab=forwardHab, attachment=eattach) yield self.tock # throttle just do one cue at a time yield self.tock - def cueDo(self, tymth=None, tock=0.0): + def cueDo(self, tymth=None, tock=None, **kwa): """ Returns doifiable Doist compatibile generator method (doer dog) to process Kevery and Tevery cues deque @@ -215,69 +255,74 @@ def cueDo(self, tymth=None, tock=0.0): """ # enter context self.wind(tymth) - self.tock = tock + self.tock = tock if tock is not None else tocking.RespondantCueTock _ = (yield self.tock) while True: - while self.cues: # iteratively process each cue in cues - cue = self.cues.pull() # self.cues.popleft() - cueKin = cue["kin"] # type or kind of cue - if cueKin in ("receipt",): # cue to receipt a received event from other pre - serder = cue["serder"] # Serder of received event for other pre - cuedKed = serder.ked - cuedPrefixer = coring.Prefixer(qb64=cuedKed["i"]) - - # If respondant configured with list of acceptable AIDs to witness for, check them here - if self.aids is not None and cuedPrefixer.qb64 not in self.aids: - continue - - if cuedPrefixer.qb64 in self.hby.kevers: - kever = self.hby.kevers[cuedPrefixer.qb64] - owits = oset(kever.wits) - if match := owits.intersection(self.hby.prefixes): - pre = match.pop() - hab = self.hby.habByPre(pre) - if hab is None: - continue - - raw = hab.receipt(serder) - rserder = serdering.SerderKERI(raw=raw) - del raw[:rserder.size] - self.postman.send(serder.pre, topic="receipt", serder=rserder, hab=hab, attachment=raw) - - elif cueKin in ("replay",): - src = cue["src"] - dest = cue["dest"] - msgs = cue["msgs"] - - hab = self.hby.habByPre(src) - if hab is None: - continue - - if dest not in self.hby.kevers: - continue - - for msg in msgs: - raw = bytearray(msg) - serder = serdering.SerderKERI(raw=raw) - del raw[:serder.size] - self.postman.send(dest, topic="replay", serder=serder, hab=hab, attachment=raw) - - elif cueKin in ("reply",): - src = cue["src"] - serder = cue["serder"] - - dest = cue["dest"] - - if dest not in self.hby.kevers: - continue - - hab = self.hby.habByPre(src) - if hab is None: - continue - - atc = hab.endorse(serder) - del atc[:serder.size] - self.postman.send(hab=hab, dest=dest, topic="reply", serder=serder, attachment=atc) + while not self.cues: + yield self.tock + + cue = self.cues.pull() # self.cues.popleft() + cueKin = cue["kin"] # type or kind of cue + if cueKin in ("receipt",): # cue to receipt a received event from other pre + serder = cue["serder"] # Serder of received event for other pre + cuedKed = serder.ked + cuedPrefixer = coring.Prefixer(qb64=cuedKed["i"]) + + # If respondant configured with list of acceptable AIDs to witness for, check them here + if self.aids is not None and cuedPrefixer.qb64 not in self.aids: + continue + + if cuedPrefixer.qb64 in self.hby.kevers: + kever = self.hby.kevers[cuedPrefixer.qb64] + owits = oset(kever.wits) + if match := owits.intersection(self.hby.prefixes): + pre = match.pop() + hab = self.hby.habByPre(pre) + if hab is None: + continue + + raw = hab.receipt(serder) + rserder = serdering.SerderKERI(raw=raw) + del raw[:rserder.size] + self.postman.send(serder.pre, topic="receipt", serder=rserder, hab=hab, attachment=raw) + + elif cueKin in ("replay",): + src = cue["src"] + dest = cue["dest"] + msgs = cue["msgs"] + + hab = self.hby.habByPre(src) + if hab is None: + continue + + if dest not in self.hby.kevers: + continue + + for msg in msgs: + raw = bytearray(msg) + serder = serdering.SerderKERI(raw=raw) + del raw[:serder.size] + self.postman.send(dest, topic="replay", serder=serder, hab=hab, attachment=raw) + + elif cueKin in ("reply",): + src = cue["src"] + serder = cue["serder"] + + dest = cue["dest"] + + if dest not in self.hby.kevers: + continue + + hab = self.hby.habByPre(src) + if hab is None: + continue + + atc = hab.endorse(serder) + del atc[:serder.size] + self.postman.send(hab=hab, dest=dest, topic="reply", serder=serder, attachment=atc) + + else: + self.cues.push(cue) yield self.tock diff --git a/src/keri/app/tocking.py b/src/keri/app/tocking.py new file mode 100644 index 000000000..a97d6264c --- /dev/null +++ b/src/keri/app/tocking.py @@ -0,0 +1,89 @@ +# -*- encoding: utf-8 -*- +""" +keri.app.tocking module + +Centralized tock configuration for doer timing control. +Environment variables allow production tuning without code changes. + +Tock values control how frequently doer methods execute: +- 0.0 = run as soon as possible (~31ms doist interval) +- 1.0 = run once per second +- Higher values = less frequent execution, lower CPU usage + +""" +import os + +# Witness component tocks (indirecting.py - WitnessStart) +KERI_WITNESS_MSG_TOCK_KEY = "KERI_WITNESS_MSG_TOCK" +KERI_WITNESS_ESCROW_TOCK_KEY = "KERI_WITNESS_ESCROW_TOCK" +KERI_WITNESS_CUE_TOCK_KEY = "KERI_WITNESS_CUE_TOCK" + +WitnessMsgTock = float(os.getenv(KERI_WITNESS_MSG_TOCK_KEY, "0.0")) +WitnessEscrowTock = float(os.getenv(KERI_WITNESS_ESCROW_TOCK_KEY, "1.0")) +WitnessCueTock = float(os.getenv(KERI_WITNESS_CUE_TOCK_KEY, "0.25")) + +# Indirector component tocks (indirecting.py - Indirector) +KERI_INDIRECTOR_MSG_TOCK_KEY = "KERI_INDIRECTOR_MSG_TOCK" +KERI_INDIRECTOR_CUE_TOCK_KEY = "KERI_INDIRECTOR_CUE_TOCK" +KERI_INDIRECTOR_ESCROW_TOCK_KEY = "KERI_INDIRECTOR_ESCROW_TOCK" + +IndirectorMsgTock = float(os.getenv(KERI_INDIRECTOR_MSG_TOCK_KEY, "0.0")) +IndirectorCueTock = float(os.getenv(KERI_INDIRECTOR_CUE_TOCK_KEY, "0.25")) +IndirectorEscrowTock = float(os.getenv(KERI_INDIRECTOR_ESCROW_TOCK_KEY, "1.0")) + +# MailboxDirector component tocks (indirecting.py - MailboxDirector) +KERI_MAILBOX_POLL_TOCK_KEY = "KERI_MAILBOX_POLL_TOCK" +KERI_MAILBOX_MSG_TOCK_KEY = "KERI_MAILBOX_MSG_TOCK" +KERI_MAILBOX_ESCROW_TOCK_KEY = "KERI_MAILBOX_ESCROW_TOCK" + +MailboxPollTock = float(os.getenv(KERI_MAILBOX_POLL_TOCK_KEY, "0.5")) +MailboxMsgTock = float(os.getenv(KERI_MAILBOX_MSG_TOCK_KEY, "0.0")) +MailboxEscrowTock = float(os.getenv(KERI_MAILBOX_ESCROW_TOCK_KEY, "1.0")) + +# Poller component tocks (indirecting.py - Poller) +KERI_POLLER_EVENT_TOCK_KEY = "KERI_POLLER_EVENT_TOCK" + +PollerEventTock = float(os.getenv(KERI_POLLER_EVENT_TOCK_KEY, "0.5")) + +# ReceiptEnd component tocks (indirecting.py - ReceiptEnd) +KERI_RECEIPT_INTERCEPT_TOCK_KEY = "KERI_RECEIPT_INTERCEPT_TOCK" + +ReceiptInterceptTock = float(os.getenv(KERI_RECEIPT_INTERCEPT_TOCK_KEY, "0.1")) + +# Reactor component tocks (directing.py - Reactor) +KERI_REACTOR_MSG_TOCK_KEY = "KERI_REACTOR_MSG_TOCK" +KERI_REACTOR_CUE_TOCK_KEY = "KERI_REACTOR_CUE_TOCK" +KERI_REACTOR_ESCROW_TOCK_KEY = "KERI_REACTOR_ESCROW_TOCK" + +ReactorMsgTock = float(os.getenv(KERI_REACTOR_MSG_TOCK_KEY, "0.0")) +ReactorCueTock = float(os.getenv(KERI_REACTOR_CUE_TOCK_KEY, "0.25")) +ReactorEscrowTock = float(os.getenv(KERI_REACTOR_ESCROW_TOCK_KEY, "1.0")) + +# Reactant component tocks (directing.py - Reactant) +KERI_REACTANT_MSG_TOCK_KEY = "KERI_REACTANT_MSG_TOCK" +KERI_REACTANT_CUE_TOCK_KEY = "KERI_REACTANT_CUE_TOCK" +KERI_REACTANT_ESCROW_TOCK_KEY = "KERI_REACTANT_ESCROW_TOCK" + +ReactantMsgTock = float(os.getenv(KERI_REACTANT_MSG_TOCK_KEY, "0.0")) +ReactantCueTock = float(os.getenv(KERI_REACTANT_CUE_TOCK_KEY, "0.25")) +ReactantEscrowTock = float(os.getenv(KERI_REACTANT_ESCROW_TOCK_KEY, "1.0")) + +# Respondant component tocks (storing.py - Respondant) +KERI_RESPONDANT_CUE_TOCK_KEY = "KERI_RESPONDANT_CUE_TOCK" + +RespondantCueTock = float(os.getenv(KERI_RESPONDANT_CUE_TOCK_KEY, "0.25")) + +# WitnessReceiptor component tocks (agenting.py) +KERI_WITNESS_RECEIPTOR_TOCK_KEY = "KERI_WITNESS_RECEIPTOR_TOCK" + +WitnessReceiptorTock = float(os.getenv(KERI_WITNESS_RECEIPTOR_TOCK_KEY, "0.0")) + +# WitnessInquisitor component tocks (agenting.py) +KERI_WITNESS_INQUISITOR_TOCK_KEY = "KERI_WITNESS_INQUISITOR_TOCK" + +WitnessInquisitorTock = float(os.getenv(KERI_WITNESS_INQUISITOR_TOCK_KEY, "1.0")) + +# WitnessPublisher component tocks (agenting.py) +KERI_WITNESS_PUBLISHER_TOCK_KEY = "KERI_WITNESS_PUBLISHER_TOCK" + +WitnessPublisherTock = float(os.getenv(KERI_WITNESS_PUBLISHER_TOCK_KEY, "0.25")) diff --git a/src/keri/app/watching.py b/src/keri/app/watching.py new file mode 100644 index 000000000..5159c0eb5 --- /dev/null +++ b/src/keri/app/watching.py @@ -0,0 +1,223 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.app.watching module + +""" +import random +from collections import namedtuple +from dataclasses import dataclass + +from hio.base import doing +from hio.help import decking + +from keri import help + +logger = help.ogler.getLogger() + +Stateage = namedtuple("Stateage", 'even ahead behind duplicitous') + +States = Stateage(even="even", ahead="ahead", behind="behind", duplicitous="duplicitous") + + +@dataclass +class DiffState: + """ Difference between a remote KeyStateRecord and local for the same AID. + + Uses Stateage to represent whether the remote KSR is even, ahead, behind or duplicitous + + """ + pre: str # The AID of the KSR + wit: str # The entity reporting the KSR (non-local) + state: Stateage # The state of the remote KSR relative to local + sn: int # The sequence number of the remote KSR + dig: str # The digest of the latest event of the remote KSR + + +class Adjudicator: + """ The Adjudicator of Key State + + This class performs key state adjudication by checking any key state reported by the watcher set for a given + watched AID and compares the reported values against the local key state for the watched AID and the key state + of all other responding watchers. It uses a per-adjudication threshold to determine what is acceptable duplicity + for each adjudication. + + Cues are sent out for each round of adjudication with the following kins: + + keyStateConsistent - Key state of all queries watchers is consistent with local key state + keyStateLagging - Key state from some watchers is behind local key state and other watchers + keyStateUpdate - A threshold satisfying number of watchers report new key state for watched AID + keyStateDuplicitous - Duplicity has been detected on some set of watchers (provided in the cue) + + Consumers of the Adjudicator's cues are safe to retrieve new key state from one of the Watchers listed in the + cue of `keyStateUpdated` is received. All other kins require controller intervention and should be bubbled up. + + """ + + def __init__(self, hby, hab, msgs=None, cues=None): + """ Create instance of Adjudicator for adjudicating key state + + Parameters: + hby (Habery): database and Habitat environment + hab (Hab): identifier database environment + msgs (Deck): incoming requests to adjudicate key state + cues (Deck): outgoing responses to adjudication of key state + + """ + self.hby = hby + self.hab = hab + self.msgs = msgs if msgs is not None else decking.Deck() + self.cues = cues if cues is not None else decking.Deck() + + def performAdjudications(self): + """ Process loop of existing messages requesting key state adjudication """ + while self.msgs: + msg = self.msgs.pull() + + watched = msg["oid"] + toad = msg["toad"] if "toad" in msg else None + + self.adjudicate(watched, toad) + + def adjudicate(self, watched, toad=None): + """ Perform key state adjudication against the `watched` AID and provided threshold + + If `toad` is not provided, the full set of watchers must come to consensus before `keyStateUpdate` + will be reported. + + Parameters: + watched (str): qb64 AID to adjudicate for key state duplicity + toad (int): threshold of acceptable duplicity amongst available watchers + + + """ + watchers = set() + for (cid, aid, oid), observed in self.hab.db.obvs.getItemIter(keys=(self.hab.pre,)): + if observed.enabled and oid == watched: + watchers.add(aid) + + toad = int(toad) if toad else len(watchers) + if toad > len(watchers): + raise ValueError(f"Threshold of {toad} is greater than number watchers {len(watchers)}") + + states = [] + mystate = self.hab.kevers[watched].state() + for watcher in watchers: + saider = self.hab.db.knas.get(keys=(watched, watcher)) + if saider is None: + logger.info(f"No key state from watcher {watcher} for {watched}") + continue + + ksn = self.hab.db.ksns.get(keys=(saider.qb64,)) + states.append(diffState(watcher, mystate, ksn)) + + dups = [state for state in states if state.state == States.duplicitous] + ahds = [state for state in states if state.state == States.ahead] + bhds = [state for state in states if state.state == States.behind] + + if len(dups) > 0: + cue = dict(kin="keyStateDuplicitous", cid=self.hab.pre, oid=watched, wids=watchers, dups=dups) + self.cues.append(cue) + + logger.error(f"Duplicity detected for AID {watched}, local key state remains intact.") + for state in dups: + logger.error(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + elif len(ahds) > 0: + # Only group habs can be behind their watchers + # First check for duplicity among the watchers that are ahead (possible only if toad is below + # super majority) + digs = set([state.dig for state in ahds]) + if len(digs) > 1: # Duplicity across watcher sets + cue = dict(kin="keyStateDuplicitous", cid=self.hab.pre, oid=watched, wids=watchers, dups=ahds) + self.cues.append(cue) + + logger.error(f"There are multiple duplicitous events on watcher for {watched}") + for state in ahds: + logger.error(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + elif len(ahds) >= toad: # all witnesses that are ahead agree on the event + logger.info(f"Threshold ({toad}) satisfying number of watchers ({len(ahds)}) are ahead") + for state in ahds: + logger.info(f"\tWatcher {state.wit} at Seq No. {state.sn} with digest: {state.dig}") + + state = random.choice(ahds) + cue = dict(kin="keyStateUpdate", cid=self.hab.pre, oid=watched, wids=watchers, sn=state.sn, aheads=ahds) + self.cues.append(cue) + + elif len(bhds) > 0: + cue = dict(kin="keyStateLagging", cid=self.hab.pre, oid=watched, wids=watchers, behind=bhds) + self.cues.append(cue) + + logger.info("The following watchers are behind the local KEL:") + for state in bhds: + logger.info(f"\tWatcher {state.wit} at seq No. {state.sn} with digest: {state.dig}") + + logger.info(f"Recommend the checking those watchers for access to {watched} witnesses") + + else: + cue = dict(kin="keyStateConsistent", cid=self.hab.pre, oid=watched, wids=watchers, states=states) + self.cues.append(cue) + logger.info(f"Local key state is consistent with the {len(states)} (out of " + f"{len(watchers)} total) watchers that responded") + + +class AdjudicationDoer(doing.Doer): + """ Doer class responsible for process adjudication requests in an Adjudicator's msgs """ + + def __init__(self, adjudicator): + """ Create instance of Doer for performing key state adjudications """ + self.adjudicator = adjudicator + super(AdjudicationDoer, self).__init__() + + def recur(self, tyme): + """ Perform one pass over all adjudication requests + + Parameters: + tyme (float): relative cycle time + + Returns: + + """ + self.adjudicator.performAdjudications() + + +def diffState(wit, preksn, witksn): + """ Return a record of the differences between the states provided by `wit` and local state + + Parameters: + wit (str): qb64 AID of entity reporting key state + preksn (KeyStateRecord): Local key state of AID + witksn (KeyStateRecord): Key state of AID as provided by `wit` + + Returns: + state (WitnessState): record indicating the differenced between the two provided KSN records + + """ + mypre = preksn.i + pre = witksn.i + mysn = int(preksn.s, 16) + mydig = preksn.d + sn = int(witksn.s, 16) + dig = witksn.d + + if pre != mypre: + raise ValueError(f"can't compare key states from different AIDs {mypre}/{pre}") + + # At the same sequence number, check the DIGs + if mysn == sn: + if mydig == dig: + state = States.even + else: + state = States.duplicitous + + # This witness is behind and will need to be caught up. + elif mysn > sn: + state = States.behind + + # mysn < witstate.sn - We are behind this witness (multisig or restore situation). + # Must ensure that controller approves this event or a recovery rotation is needed + else: + state = States.ahead + + return DiffState(pre, wit, state, sn, dig) diff --git a/src/keri/core/__init__.py b/src/keri/core/__init__.py index 02761a160..b37b9516c 100644 --- a/src/keri/core/__init__.py +++ b/src/keri/core/__init__.py @@ -4,4 +4,16 @@ keri.core Package """ -__all__ = ["coring", "eventing", "parsing", "scheming"] +# Constants etc +from .coring import (Tiers, ) + +# Matter class and its subclasses +from .coring import (Matter, MtrDex, Number, NumDex, Dater, Texter, + Bexter, Pather, Verfer, Cigar, Diger, DigDex, + Prefixer, PreDex, ) + +from .coring import Tholder +from .indexing import Indexer, Siger, IdrDex, IdxSigDex +from .signing import Signer, Salter, Cipher, CiXDex, Encrypter, Decrypter +from .counting import Counter, Codens, CtrDex_2_0 +from .streaming import Streamer diff --git a/src/keri/core/coring.py b/src/keri/core/coring.py index 4d58794c3..99fe7e18a 100644 --- a/src/keri/core/coring.py +++ b/src/keri/core/coring.py @@ -6,10 +6,9 @@ import re import json from typing import Union -from collections.abc import Iterable - -from dataclasses import dataclass, astuple from collections import namedtuple, deque +from collections.abc import Sequence, Mapping +from dataclasses import dataclass, astuple, asdict from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 from fractions import Fraction @@ -25,7 +24,11 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.hazmat.primitives.asymmetric import ec, utils -from ..kering import (EmptyMaterialError, RawMaterialError, InvalidCodeError, +from ..kering import MaxON + +from ..kering import (EmptyMaterialError, RawMaterialError, SoftMaterialError, + InvalidCodeError, InvalidSoftError, + InvalidSizeError, InvalidCodeSizeError, InvalidVarIndexError, InvalidVarSizeError, InvalidVarRawSizeError, ConversionError, InvalidValueError, InvalidTypeError, @@ -33,33 +36,28 @@ EmptyListError, ShortageError, UnexpectedCodeError, DeserializeError, UnexpectedCountCodeError, UnexpectedOpCodeError) -from ..kering import (Versionage, Version, VERRAWSIZE, VERFMT, VERFULLSIZE, - versify, deversify, Rever) -from ..kering import Serials, Serialage, Protos, Protocolage, Ilkage, Ilks -from ..kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, - RPY_LABELS) -from ..kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, - BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) +from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, + VERRAWSIZE, VERFMT, MAXVERFULLSPAN, + versify, deversify, Rever, smell) +from ..kering import (Kinds, Kindage, Protocols, Protocolage, Ilkage, Ilks, + TraitDex, ) from ..help import helping -from ..help.helping import sceil, nonStringIterable +from ..help.helping import sceil, nonStringIterable, nonStringSequence +from ..help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) + -Labels = Ilkage(icp=ICP_LABELS, rot=ROT_LABELS, ixn=IXN_LABELS, dip=DIP_LABELS, - drt=DRT_LABELS, rct=[], qry=[], rpy=RPY_LABELS, - exn=[], pro=[], bar=[], - vcp=VCP_LABELS, vrt=VRT_LABELS, iss=ISS_LABELS, rev=REV_LABELS, - bis=BIS_LABELS, brv=BRV_LABELS) DSS_SIG_MODE = "fips-186-3" ECDSA_256r1_SEEDBYTES = 32 ECDSA_256k1_SEEDBYTES = 32 - -Vstrings = Serialage(json=versify(kind=Serials.json, size=0), - mgpk=versify(kind=Serials.mgpk, size=0), - cbor=versify(kind=Serials.cbor, size=0)) +# digest algorithm klas, digest size (not default), digest length +# size and length are needed for some digest types as function parameters +Digestage = namedtuple("Digestage", "klas size length") # SAID field labels Saidage = namedtuple("Saidage", "dollar at id_ i d") @@ -91,7 +89,7 @@ def sizeify(ked, kind=None, version=Version): raise ValueError("Missing or empty version string in key event " "dict = {}".format(ked)) - proto, vrsn, knd, size = deversify(ked["v"]) # extract kind and version + proto, vrsn, knd, size, _ = deversify(ked["v"]) # extract kind and version if vrsn != version: raise ValueError("Unsupported version = {}.{}".format(vrsn.major, vrsn.minor)) @@ -99,7 +97,7 @@ def sizeify(ked, kind=None, version=Version): if not kind: kind = knd - if kind not in Serials: + if kind not in Kinds: raise ValueError("Invalid serialization kind = {}".format(kind)) raw = dumps(ked, kind) @@ -111,7 +109,7 @@ def sizeify(ked, kind=None, version=Version): fore, back = match.span() # full version string # update vs with latest kind version size - vs = versify(proto=proto, version=vrsn, kind=kind, size=size) + vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) # replace old version string in raw with new one raw = b'%b%b%b' % (raw[:fore], vs.encode("utf-8"), raw[back:]) if size != len(raw): # substitution messed up @@ -121,151 +119,9 @@ def sizeify(ked, kind=None, version=Version): return raw, proto, kind, ked, vrsn -# Base64 utilities -BASE64_PAD = b'=' - -# Mappings between Base64 Encode Index and Decode Characters -# B64ChrByIdx is dict where each key is a B64 index and each value is the B64 char -# B64IdxByChr is dict where each key is a B64 char and each value is the B64 index -# Map Base64 index to char -B64ChrByIdx = dict((index, char) for index, char in enumerate([chr(x) for x in range(65, 91)])) -B64ChrByIdx.update([(index + 26, char) for index, char in enumerate([chr(x) for x in range(97, 123)])]) -B64ChrByIdx.update([(index + 52, char) for index, char in enumerate([chr(x) for x in range(48, 58)])]) -B64ChrByIdx[62] = '-' -B64ChrByIdx[63] = '_' -# Map char to Base64 index -B64IdxByChr = {char: index for index, char in B64ChrByIdx.items()} -B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 - -B64REX = b'^[A-Za-z0-9\-\_]*\Z' -Reb64 = re.compile(B64REX) # compile is faster - - -def intToB64(i, l=1): - """ - Returns conversion of int i to Base64 str - l is min number of b64 digits left padded with Base64 0 == "A" char - """ - d = deque() # deque of characters base64 - - while l: - d.appendleft(B64ChrByIdx[i % 64]) - i = i // 64 - if not i: - break - # d.appendleft(B64ChrByIdx[i % 64]) - # i = i // 64 - for j in range(l - len(d)): # range(x) x <= 0 means do not iterate - d.appendleft("A") - return ("".join(d)) - - -def intToB64b(i, l=1): - """ - Returns conversion of int i to Base64 bytes - l is min number of b64 digits left padded with Base64 0 == "A" char - """ - return (intToB64(i=i, l=l).encode("utf-8")) - - -def b64ToInt(s): - """ - Returns conversion of Base64 str s or bytes to int - """ - if not s: - raise ValueError("Empty string, conversion undefined.") - if hasattr(s, 'decode'): - s = s.decode("utf-8") - i = 0 - for e, c in enumerate(reversed(s)): - i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) - return i - - -def codeB64ToB2(s): - """ - Returns conversion (decode) of Base64 chars to Base2 bytes. - Where the number of total bytes returned is equal to the minimun number of - octets sufficient to hold the total converted concatenated sextets from s, - with one sextet per each Base64 decoded char of s. Assumes no pad chars in s. - Sextets are left aligned with pad bits in last (rightmost) byte. - This is useful for decoding as bytes, code characters from the front of - a Base64 encoded string of characters. - """ - i = b64ToInt(s) - i <<= 2 * (len(s) % 4) # add 2 bits right zero padding for each sextet - n = sceil(len(s) * 3 / 4) # compute min number of ocetets to hold all sextets - return (i.to_bytes(n, 'big')) - - -def codeB2ToB64(b, l): - """ - Returns conversion (encode) of l Base2 sextets from front of b to Base64 chars. - One char for each of l sextets from front (left) of b. - This is useful for encoding as code characters, sextets from the front of - a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 - and l=4. Both require 3 bytes in b. - """ - if hasattr(b, 'encode'): - b = b.encode("utf-8") # convert to bytes - n = sceil(l * 3 / 4) # number of bytes needed for l sextets - if n > len(b): - raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) - i = int.from_bytes(b[:n], 'big') # convert only first n bytes to int - # check if prepad bits are zero - tbs = 2 * (l % 4) # trailing bit size in bits - i >>= tbs # right shift out trailing bits to make right aligned - return (intToB64(i, l)) # return as B64 - - -def nabSextets(b, l): - """ - Return first l sextets from front (left) of b as bytes (byte string). - Length of bytes returned is minimum sufficient to hold all l sextets. - Last byte returned is right bit padded with zeros - b is bytes or str - """ - if hasattr(b, 'encode'): - b = b.encode("utf-8") # convert to bytes - n = sceil(l * 3 / 4) # number of bytes needed for l sextets - if n > len(b): - raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) - i = int.from_bytes(b[:n], 'big') - p = 2 * (l % 4) - i >>= p # strip of last bits - i <<= p # pad with empty bits - return (i.to_bytes(n, 'big')) - -MINSNIFFSIZE = 12 + VERFULLSIZE # min bytes in buffer to sniff else need more - -def sniff(raw): - """ - Returns serialization kind, version and size from serialized event raw - by investigating leading bytes that contain version string - - Parameters: - raw is bytes of serialized event - - """ - if len(raw) < MINSNIFFSIZE: - raise ShortageError("Need more bytes.") - - match = Rever.search(raw) # Rever's regex takes bytes - if not match or match.start() > 12: - raise VersionError("Invalid version string in raw = {}".format(raw)) - - proto, major, minor, kind, size = match.group("proto", "major", "minor", "kind", "size") - version = Versionage(major=int(major, 16), minor=int(minor, 16)) - kind = kind.decode("utf-8") - proto = proto.decode("utf-8") - if kind not in Serials: - raise DeserializeError("Invalid serialization kind = {}".format(kind)) - size = int(size, 16) - - return proto, kind, version, size -def dumps(ked, kind=Serials.json): +def dumps(ked, kind=Kinds.json): """ utility function to handle serialization by kind @@ -276,13 +132,13 @@ def dumps(ked, kind=Serials.json): ked (Optional(dict, list)): key event dict or message dict to serialize kind (str): serialization kind (JSON, MGPK, CBOR) """ - if kind == Serials.json: + if kind == Kinds.json: raw = json.dumps(ked, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: raw = msgpack.dumps(ked) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: raw = cbor.dumps(ked) else: raise ValueError("Invalid serialization kind = {}".format(kind)) @@ -290,7 +146,7 @@ def dumps(ked, kind=Serials.json): return raw -def loads(raw, size=None, kind=Serials.json): +def loads(raw, size=None, kind=Kinds.json): """ utility function to handle deserialization by kind @@ -303,21 +159,21 @@ def loads(raw, size=None, kind=Serials.json): then consume all bytes kind (str): serialization kind (JSON, MGPK, CBOR) """ - if kind == Serials.json: + if kind == Kinds.json: try: ked = json.loads(raw[:size].decode("utf-8")) except Exception as ex: raise DeserializeError("Error deserializing JSON: {}" "".format(raw[:size].decode("utf-8"))) - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: try: ked = msgpack.loads(raw[:size]) except Exception as ex: raise DeserializeError("Error deserializing MGPK: {}" "".format(raw[:size])) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: try: ked = cbor.loads(raw[:size]) except Exception as ex: @@ -331,71 +187,91 @@ def loads(raw, size=None, kind=Serials.json): return ked -def generateSigners(salt=None, count=8, transferable=True): - """ - Returns list of Signers for Ed25519 +# Deprecated +# randomNonce() refactored to match Salter().qb64 and only used in coring to avoid circular dependencies +# use Salter().qb64 in other places - Parameters: - salt is bytes 16 byte long root cryptomatter from which seeds for Signers - in list are derived - random salt created if not provided - count is number of signers in list - transferable is boolean true means signer.verfer code is transferable - non-transferable otherwise +def randomNonce(): + """ Generate a random 128 bits salt and encode as qb64 + + Returns: + str: qb64 encoded 128 bits random salt """ - if not salt: - salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + preseed = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + seedqb64 = Matter(raw=preseed, code=MtrDex.Salt_128).qb64 + return seedqb64 - signers = [] - for i in range(count): - path = f"{i:x}" - # algorithm default is argon2id - seed = pysodium.crypto_pwhash(outlen=32, - passwd=path, - salt=salt, - opslimit=2, # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE, - memlimit=67108864, # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE, - alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) - signers.append(Signer(raw=seed, transferable=transferable)) +# secret derivation security tier +Tierage = namedtuple("Tierage", 'low med high') - return signers +Tiers = Tierage(low='low', med='med', high='high') -def generatePrivates(salt=None, count=8): - """ - Returns list of fully qualified Base64 secret Ed25519 seeds i.e private keys - Parameters: - salt is bytes 16 byte long root cryptomatter from which seeds for Signers - in list are derived - random salt created if not provided - count is number of signers in list +@dataclass +class MapHood: + """Base class for mutable dataclasses that support map syntax + Adds support for dunder methods for map syntax dc[name]. + Converts exceptions from attribute syntax to raise map syntax when using + map syntax. + + Enables dataclass instances to use Mapping item syntax """ - signers = generateSigners(salt=salt, count=count) - return [signer.qb64 for signer in signers] # fetch sigkey as private key + def __getitem__(self, name): + try: + return getattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex -def generatePublics(salt=None, count=8, transferable=True): - """ - Returns list of fully qualified Base64 secret seeds for Ed25519 private keys + def __setitem__(self, name, value): + try: + return setattr(self, name, value) + except AttributeError as ex: + raise IndexError(ex.args) from ex - Parameters: - salt is bytes 16 byte long root cryptomatter from which seeds for Signers - in list are derived - random salt created if not provided - count is number of signers in list + + def __delitem__(self, name): + try: + return delattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + +@dataclass(frozen=True) +class MapDom: + """Base class for frozen dataclasses (codexes) that support map syntax + Adds support for dunder methods for map syntax dc[name]. + Converts exceptions from attribute syntax to raise map syntax when using + map syntax. + + Enables dataclass instances to use Mapping item syntax """ - signers = generateSigners(salt=salt, count=count, transferable=transferable) - return [signer.verfer.qb64 for signer in signers] # fetch verkey as public key + def __getitem__(self, name): + try: + return getattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __setitem__(self, name, value): + try: + return setattr(self, name, value) + except AttributeError as ex: + raise IndexError(ex.args) from ex + + + def __delitem__(self, name): + try: + return delattr(self, name) + except AttributeError as ex: + raise IndexError(ex.args) from ex -# secret derivation security tier -Tierage = namedtuple("Tierage", 'low med high') -Tiers = Tierage(low='low', med='med', high='high') @dataclass(frozen=True) @@ -408,7 +284,7 @@ class MatterCodex: Ed25519_Seed: str = 'A' # Ed25519 256 bit random seed for private key Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - X25519: str = 'C' # X25519 public encryption key, converted from Ed25519 or Ed25519N. + X25519: str = 'C' # X25519 public encryption key, may be converted from Ed25519 or Ed25519N. Ed25519: str = 'D' # Ed25519 verification key basic derivation Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. @@ -420,18 +296,19 @@ class MatterCodex: X448: str = 'L' # X448 public encryption key, converted from Ed448 Short: str = 'M' # Short 2 byte b2 number Big: str = 'N' # Big 8 byte b2 number - X25519_Private: str = 'O' # X25519 private decryption key converted from Ed25519 + X25519_Private: str = 'O' # X25519 private decryption key/seed, may be converted from Ed25519 X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed ECDSA_256r1_Seed: str = "Q" # ECDSA secp256r1 256 bit random Seed for private key Tall: str = 'R' # Tall 5 byte b2 number Large: str = 'S' # Large 11 byte b2 number Great: str = 'T' # Great 14 byte b2 number Vast: str = 'U' # Vast 17 byte b2 number - Label1: str = 'V' # Label1 as one char (bytes) field map label lead size 1 - Label2: str = 'W' # Label2 as two char (bytes) field map label lead size 0 - Tag3: str = 'X' # Tag3 3 B64 encoded chars for field tag or packet type, semver, trait like 'DND' - Tag7: str = 'Y' # Tag7 7 B64 encoded chars for field tag or packet kind and version KERIVVV - Salt_128: str = '0A' # 128 bit random salt or 128 bit number (see Huge) + Label1: str = 'V' # Label1 1 bytes for label lead size 1 + Label2: str = 'W' # Label2 2 bytes for label lead size 0 + Tag3: str = 'X' # Tag3 3 B64 encoded chars for special values + Tag7: str = 'Y' # Tag7 7 B64 encoded chars for special values + Blind: str = 'Z' # Blinding factor 256 bits, Cryptographic strength deterministically generated from random salt + Salt_128: str = '0A' # random salt/seed/nonce/private key or number of length 128 bits (Huge) Ed25519_Sig: str = '0B' # Ed25519 signature. ECDSA_256k1_Sig: str = '0C' # ECDSA secp256k1 signature. Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. @@ -440,25 +317,32 @@ class MatterCodex: SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. Long: str = '0H' # Long 4 byte b2 number ECDSA_256r1_Sig: str = '0I' # ECDSA secp256r1 signature. - Tag1: str = '0J' # Tag1 1 B64 encoded char with pre pad for field tag - Tag2: str = '0K' # Tag2 2 B64 encoded chars for field tag or version VV or trait like 'EO' - Tag5: str = '0L' # Tag5 5 B64 encoded chars with pre pad for field tag - Tag6: str = '0M' # Tag6 6 B64 encoded chars for field tag or protocol kind version like KERIVV (KERI 1.1) or KKKVVV + Tag1: str = '0J' # Tag1 1 B64 encoded char + 1 prepad for special values + Tag2: str = '0K' # Tag2 2 B64 encoded chars for for special values + Tag5: str = '0L' # Tag5 5 B64 encoded chars + 1 prepad for special values + Tag6: str = '0M' # Tag6 6 B64 encoded chars for special values + Tag9: str = '0N' # Tag9 9 B64 encoded chars + 1 prepad for special values + Tag10: str = '0O' # Tag10 10 B64 encoded chars for special values ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. Ed448: str = '1AAD' # Ed448 public signing verification key. Basic derivation. Ed448_Sig: str = '1AAE' # Ed448 signature. Self-signing derivation. - Tag4: str = '1AAF' # Tag4 4 B64 encoded chars for field tag or message kind + Tag4: str = '1AAF' # Tag4 4 B64 encoded chars for special values DateTime: str = '1AAG' # Base64 custom encoded 32 char ISO-8601 DateTime X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt ECDSA_256r1N: str = '1AAI' # ECDSA secp256r1 verification key non-transferable, basic derivation. ECDSA_256r1: str = '1AAJ' # ECDSA secp256r1 verification or encryption key, basic derivation Null: str = '1AAK' # Null None or empty value - Yes: str = '1AAL' # Yes Truthy Boolean value - No: str = '1AAM' # No Falsey Boolean value - TBD1: str = '2AAA' # Testing purposes only fixed with lead size 1 - TBD2: str = '3AAA' # Testing purposes only of fixed with lead size 2 + No: str = '1AAL' # No Falsey Boolean value + Yes: str = '1AAM' # Yes Truthy Boolean value + Tag8: str = '1AAN' # Tag8 8 B64 encoded chars for special values + TBD0S: str = '1__-' # Testing purposes only, fixed special values with non-empty raw lead size 0 + TBD0: str = '1___' # Testing purposes only, fixed with lead size 0 + TBD1S: str = '2__-' # Testing purposes only, fixed special values with non-empty raw lead size 1 + TBD1: str = '2___' # Testing purposes only, fixed with lead size 1 + TBD2S: str = '3__-' # Testing purposes only, fixed special values with non-empty raw lead size 2 + TBD2: str = '3___' # Testing purposes only, fixed with lead size 2 StrB64_L0: str = '4A' # String Base64 only lead size 0 StrB64_L1: str = '5A' # String Base64 only lead size 1 StrB64_L2: str = '6A' # String Base64 only lead size 2 @@ -471,24 +355,24 @@ class MatterCodex: Bytes_Big_L0: str = '7AAB' # Byte String big lead size 0 Bytes_Big_L1: str = '8AAB' # Byte String big lead size 1 Bytes_Big_L2: str = '9AAB' # Byte String big lead size 2 - X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 - X25519_Cipher_QB2_L0: str = '4D' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 - X25519_Cipher_QB2_L1: str = '5D' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 - X25519_Cipher_QB2_L2: str = '6D' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 - X25519_Cipher_QB2_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 - X25519_Cipher_QB2_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 - X25519_Cipher_QB2_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 def __iter__(self): @@ -498,6 +382,8 @@ def __iter__(self): MtrDex = MatterCodex() # Make instance + + @dataclass(frozen=True) class SmallVarRawSizeCodex: """ @@ -540,23 +426,50 @@ def __iter__(self): LargeVrzDex = LargeVarRawSizeCodex() # Make instance + @dataclass(frozen=True) -class NonTransCodex: +class BextCodex: """ - NonTransCodex is codex all non-transferable derivation codes + BextCodex is codex of all variable sized Base64 Text (Bext) derivation codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. - Ed448N: str = '1AAC' # Ed448 non-transferable prefix public signing verification key. Basic derivation. - ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. + StrB64_L0: str = '4A' # String Base64 Only Leader Size 0 + StrB64_L1: str = '5A' # String Base64 Only Leader Size 1 + StrB64_L2: str = '6A' # String Base64 Only Leader Size 2 + StrB64_Big_L0: str = '7AAA' # String Base64 Only Big Leader Size 0 + StrB64_Big_L1: str = '8AAA' # String Base64 Only Big Leader Size 1 + StrB64_Big_L2: str = '9AAA' # String Base64 Only Big Leader Size 2 def __iter__(self): return iter(astuple(self)) -NonTransDex = NonTransCodex() # Make instance +BexDex = BextCodex() # Make instance + + + +@dataclass(frozen=True) +class TextCodex: + """ + TextCodex is codex of all variable sized byte string (Text) derivation codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Bytes_L0: str = '4B' # Byte String lead size 0 + Bytes_L1: str = '5B' # Byte String lead size 1 + Bytes_L2: str = '6B' # Byte String lead size 2 + Bytes_Big_L0: str = '7AAB' # Byte String big lead size 0 + Bytes_Big_L1: str = '8AAB' # Byte String big lead size 1 + Bytes_Big_L2: str = '9AAB' # Byte String big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + + +TexDex = TextCodex() # Make instance + + # When add new to DigCodes update Saider.Digests and Serder.Digests class attr @dataclass(frozen=True) @@ -596,8 +509,12 @@ class NumCodex: """ Short: str = 'M' # Short 2 byte b2 number Long: str = '0H' # Long 4 byte b2 number + Tall: str = 'R' # Tall 5 byte b2 number Big: str = 'N' # Big 8 byte b2 number + Large: str = 'S' # Large 11 byte b2 number + Great: str = 'T' # Great 14 byte b2 number Huge: str = '0A' # Huge 16 byte b2 number (same as Salt_128) + Vast: str = 'U' # Vast 17 byte b2 number def __iter__(self): return iter(astuple(self)) @@ -606,37 +523,59 @@ def __iter__(self): NumDex = NumCodex() # Make instance - - @dataclass(frozen=True) -class BextCodex: +class TagCodex: """ - BextCodex is codex of all variable sized Base64 Text (Bext) derivation codes. + TagCodex is codex of Base64 derivation codes for compactly representing + various small Base64 tag values as special code soft part values. + Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - StrB64_L0: str = '4A' # String Base64 Only Leader Size 0 - StrB64_L1: str = '5A' # String Base64 Only Leader Size 1 - StrB64_L2: str = '6A' # String Base64 Only Leader Size 2 - StrB64_Big_L0: str = '7AAA' # String Base64 Only Big Leader Size 0 - StrB64_Big_L1: str = '8AAA' # String Base64 Only Big Leader Size 1 - StrB64_Big_L2: str = '9AAA' # String Base64 Only Big Leader Size 2 + Tag1: str = '0J' # 1 B64 char tag with 1 pre pad + Tag2: str = '0K' # 2 B64 char tag + Tag3: str = 'X' # 3 B64 char tag + Tag4: str = '1AAF' # 4 B64 char tag + Tag5: str = '0L' # 5 B64 char tag with 1 pre pad + Tag6: str = '0M' # 6 B64 char tag + Tag7: str = 'Y' # 7 B64 char tag + Tag8: str = '1AAN' # 8 B64 char tag + Tag9: str = '0N' # 9 B64 char tag with 1 pre pad + Tag10: str = '0O' # 10 B64 char tag def __iter__(self): return iter(astuple(self)) -BexDex = BextCodex() # Make instance - +TagDex = TagCodex() # Make instance @dataclass(frozen=True) -class TextCodex: +class LabelCodex: """ - TextCodex is codex of all variable sized byte string (Text) derivation codes. + LabelCodex is codex of. + Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ + Tag1: str = '0J' # 1 B64 char tag with 1 pre pad + Tag2: str = '0K' # 2 B64 char tag + Tag3: str = 'X' # 3 B64 char tag + Tag4: str = '1AAF' # 4 B64 char tag + Tag5: str = '0L' # 5 B64 char tag with 1 pre pad + Tag6: str = '0M' # 6 B64 char tag + Tag7: str = 'Y' # 7 B64 char tag + Tag8: str = '1AAN' # 8 B64 char tag + Tag9: str = '0N' # 9 B64 char tag with 1 pre pad + Tag10: str = '0O' # 10 B64 char tag + StrB64_L0: str = '4A' # String Base64 Only Leader Size 0 + StrB64_L1: str = '5A' # String Base64 Only Leader Size 1 + StrB64_L2: str = '6A' # String Base64 Only Leader Size 2 + StrB64_Big_L0: str = '7AAA' # String Base64 Only Big Leader Size 0 + StrB64_Big_L1: str = '8AAA' # String Base64 Only Big Leader Size 1 + StrB64_Big_L2: str = '9AAA' # String Base64 Only Big Leader Size 2 + Label1: str = 'V' # Label1 1 bytes for label lead size 1 + Label2: str = 'W' # Label2 2 bytes for label lead size 0 Bytes_L0: str = '4B' # Byte String lead size 0 Bytes_L1: str = '5B' # Byte String lead size 1 Bytes_L2: str = '6B' # Byte String lead size 2 @@ -648,114 +587,84 @@ def __iter__(self): return iter(astuple(self)) -TexDex = TextCodex() # Make instance - -@dataclass(frozen=True) -class CipherX25519VarCodex: - """ - CipherX25519VarCodex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B2. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_L0: str = '4D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 0 - X25519_Cipher_L1: str = '5D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 1 - X25519_Cipher_L2: str = '6D' # X25519 sealed box cipher bytes of sniffable plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of sniffable plaintext big lead size 2 - - def __iter__(self): - return iter(astuple(self)) - - -CiXVarDex = CipherX25519VarCodex() # Make instance - - -@dataclass(frozen=True) -class CipherX25519FixQB64Codex: - """ - CipherX25519FixQB64Codex is codex all fixed sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B64. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed - X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt - - def __iter__(self): - return iter(astuple(self)) - +LabelDex = LabelCodex() # Make instance -CiXFixQB64Dex = CipherX25519FixQB64Codex() # Make instance @dataclass(frozen=True) -class CipherX25519VarQB64Codex: +class PreCodex: """ - CipherX25519VarQB64Codex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is QB64. + PreCodex is codex all identifier prefix derivation codes. + This is needed to verify valid inception events. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 - X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 - X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 - X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 - X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 - X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verification key, basic derivation. + Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. + Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. + Blake2s_256: str = 'G' # Blake2s 256 bit digest self-addressing derivation. + SHA3_256: str = 'H' # SHA3 256 bit digest self-addressing derivation. + SHA2_256: str = 'I' # SHA2 256 bit digest self-addressing derivation. + Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. + Blake2b_512: str = '0E' # Blake2b 512 bit digest self-addressing derivation. + SHA3_512: str = '0F' # SHA3 512 bit digest self-addressing derivation. + SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. + ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. + ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation + Ed448N: str = '1AAC' # Ed448 verification key non-transferable, basic derivation. + Ed448: str = '1AAD' # Ed448 verification key, basic derivation. + Ed448_Sig: str = '1AAE' # Ed448 signature. Self-signing derivation. + ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. + ECDSA_256r1: str = "1AAJ" # ECDSA secp256r1 verification or encryption key, basic derivation def __iter__(self): return iter(astuple(self)) -CiXVarQB64Dex = CipherX25519VarQB64Codex() # Make instance +PreDex = PreCodex() # Make instance @dataclass(frozen=True) -class CipherX25519AllQB64Codex: +class NonTransCodex: """ - CipherX25519AllQB64Codex is codex all both fixed and variable sized cipher bytes - derivation codes for sealed box encryped ciphertext. Plaintext is B64. + NonTransCodex is codex all non-transferable derivation codes Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed - X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt - X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 - X25519_Cipher_QB64_L1: str = '5E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 - X25519_Cipher_QB64_L2: str = '6E' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 - X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 - X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 - X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. + ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. + Ed448N: str = '1AAC' # Ed448 verification key non-transferable, basic derivation. + ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. def __iter__(self): return iter(astuple(self)) -CiXAllQB64Dex = CipherX25519AllQB64Codex() # Make instance +NonTransDex = NonTransCodex() # Make instance @dataclass(frozen=True) -class CipherX25519QB2VarCodex: +class PreNonDigCodex: """ - CipherX25519QB2VarCodex is codex all variable sized cipher bytes derivation codes - for sealed box encryped ciphertext. Plaintext is B2. + PreNonDigCodex is codex all prefixive but non-digestive derivation codes Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - X25519_Cipher_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 - X25519_Cipher_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 - X25519_Cipher_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 - X25519_Cipher_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 - X25519_Cipher_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 - X25519_Cipher_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. + Ed25519: str = 'D' # Ed25519 verification key, basic derivation. + ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. + ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation + Ed448N: str = '1AAC' # Ed448 verification key non-transferable, basic derivation. + Ed448: str = '1AAD' # Ed448 verification key, basic derivation. + ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. + ECDSA_256r1: str = "1AAJ" # ECDSA secp256r1 verification or encryption key, basic derivation def __iter__(self): return iter(astuple(self)) -CiXVarQB2Dex = CipherX25519QB2VarCodex() # Make instance +PreNonDigDex = PreNonDigCodex() # Make instance @@ -763,9 +672,10 @@ def __iter__(self): # namedtuple for size entries in Matter and Counter derivation code tables # hs is the hard size int number of chars in hard (stable) part of code # ss is the soft size int number of chars in soft (unstable) part of code +# xs is the xtra size into number of xtra (pre-pad) chars as part of soft # fs is the full size int number of chars in code plus appended material if any # ls is the lead size int number of bytes to pre-pad pre-converted raw binary -Sizage = namedtuple("Sizage", "hs ss fs ls") +Sizage = namedtuple("Sizage", "hs ss xs fs ls") class Matter: @@ -777,40 +687,60 @@ class Matter: Includes the following attributes and properties: + Class Attributes: + Codex (MatterCodex): MtrDex + Hards (dict): hard sizes keyed by qb64 selector + Bards (dict): hard size keyed by qb2 selector + Sizes (dict): sizes tables for codes + Codes (dict): maps code name to code + Names (dict): maps code to code name + Pad (str): B64 pad char for xtra size pre-padded soft values + + Class Methods: + + Attributes: Properties: code (str): hard part of derivation code to indicate cypher suite - both (int): hard and soft parts of full text code - size (int): Number of triplets of bytes including lead bytes - (quadlets of chars) of variable sized material. Value of soft size, - ss, part of full text code. - Otherwise None. - rize (int): number of bytes of raw material not including - lead bytes - raw (bytes): crypto material only without code + hard (str): hard part of derivation code. alias for code + soft (str | bytes): soft part of full code exclusive of xs xtra prepad. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. qb64 (str): Base64 fully qualified with derivation code + crypto mat qb64b (bytes): Base64 fully qualified with derivation code + crypto mat qb2 (bytes): binary with derivation code + crypto material transferable (bool): True means transferable derivation code False otherwise digestive (bool): True means digest derivation code False otherwise prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip Hidden: _code (str): value for .code property + _soft (str): soft value of full code _raw (bytes): value for .raw property - _rsize (bytes): value for .rsize property. Raw size in bytes when - variable sized material else None. - _size (int): value for .size property. Number of triplets of bytes - including lead bytes (quadlets of chars) of variable sized material - else None. - _infil (types.MethodType): creates qb64b from .raw and .code - (fully qualified Base64) - _exfil (types.MethodType): extracts .code and .raw from qb64b - (fully qualified Base64) + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) + + + Special soft values are indicated when fn in table is None and ss > 0. """ - Codex = MtrDex + Codex = MtrDex # class variable holding MatterDex reference + # Hards table maps from bytes Base64 first code char to int of hard size, hs, # (stable) of code. The soft size, ss, (unstable) is always 0 for Matter # unless fs is None which allows for variable size multiple of 4, i.e. @@ -819,141 +749,168 @@ class Matter: Hards.update({chr(c): 1 for c in range(97, 97 + 26)}) Hards.update([('0', 2), ('1', 4), ('2', 4), ('3', 4), ('4', 2), ('5', 2), ('6', 2), ('7', 4), ('8', 4), ('9', 4)]) - # Sizes table maps from value of hs chars of code to Sizage namedtuple of - # (hs, ss, fs, ls) where hs is hard size, ss is soft size, and fs is full size - # and ls is lead size - # soft size, ss, should always be 0 for Matter unless fs is None which allows - # for variable size multiple of 4, i.e. not (hs + ss) % 4. - Sizes = { - 'A': Sizage(hs=1, ss=0, fs=44, ls=0), - 'B': Sizage(hs=1, ss=0, fs=44, ls=0), - 'C': Sizage(hs=1, ss=0, fs=44, ls=0), - 'D': Sizage(hs=1, ss=0, fs=44, ls=0), - 'E': Sizage(hs=1, ss=0, fs=44, ls=0), - 'F': Sizage(hs=1, ss=0, fs=44, ls=0), - 'G': Sizage(hs=1, ss=0, fs=44, ls=0), - 'H': Sizage(hs=1, ss=0, fs=44, ls=0), - 'I': Sizage(hs=1, ss=0, fs=44, ls=0), - 'J': Sizage(hs=1, ss=0, fs=44, ls=0), - 'K': Sizage(hs=1, ss=0, fs=76, ls=0), - 'L': Sizage(hs=1, ss=0, fs=76, ls=0), - 'M': Sizage(hs=1, ss=0, fs=4, ls=0), - 'N': Sizage(hs=1, ss=0, fs=12, ls=0), - 'O': Sizage(hs=1, ss=0, fs=44, ls=0), - 'P': Sizage(hs=1, ss=0, fs=124, ls=0), - 'Q': Sizage(hs=1, ss=0, fs=44, ls=0), - 'R': Sizage(hs=1, ss=0, fs=8, ls=0), - 'S': Sizage(hs=1, ss=0, fs=16, ls=0), - 'T': Sizage(hs=1, ss=0, fs=20, ls=0), - 'U': Sizage(hs=1, ss=0, fs=24, ls=0), - 'V': Sizage(hs=1, ss=0, fs=4, ls=1), - 'W': Sizage(hs=1, ss=0, fs=4, ls=0), - 'X': Sizage(hs=1, ss=0, fs=4, ls=0), - 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - '0A': Sizage(hs=2, ss=0, fs=24, ls=0), - '0B': Sizage(hs=2, ss=0, fs=88, ls=0), - '0C': Sizage(hs=2, ss=0, fs=88, ls=0), - '0D': Sizage(hs=2, ss=0, fs=88, ls=0), - '0E': Sizage(hs=2, ss=0, fs=88, ls=0), - '0F': Sizage(hs=2, ss=0, fs=88, ls=0), - '0G': Sizage(hs=2, ss=0, fs=88, ls=0), - '0H': Sizage(hs=2, ss=0, fs=8, ls=0), - '0I': Sizage(hs=2, ss=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=0, fs=4, ls=0), - '0K': Sizage(hs=2, ss=0, fs=4, ls=0), - '0L': Sizage(hs=2, ss=0, fs=8, ls=0), - '0M': Sizage(hs=2, ss=0, fs=8, ls=0), - '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAD': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAE': Sizage(hs=4, ss=0, fs=56, ls=0), - '1AAF': Sizage(hs=4, ss=0, fs=8, ls=0), - '1AAG': Sizage(hs=4, ss=0, fs=36, ls=0), - '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), - '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), - '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), - '4A': Sizage(hs=2, ss=2, fs=None, ls=0), - '5A': Sizage(hs=2, ss=2, fs=None, ls=1), - '6A': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAA': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAA': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAA': Sizage(hs=4, ss=4, fs=None, ls=2), - '4B': Sizage(hs=2, ss=2, fs=None, ls=0), - '5B': Sizage(hs=2, ss=2, fs=None, ls=1), - '6B': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAB': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAB': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAB': Sizage(hs=4, ss=4, fs=None, ls=2), - '4C': Sizage(hs=2, ss=2, fs=None, ls=0), - '5C': Sizage(hs=2, ss=2, fs=None, ls=1), - '6C': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAC': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAC': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAC': Sizage(hs=4, ss=4, fs=None, ls=2), - '4D': Sizage(hs=2, ss=2, fs=None, ls=0), - '5D': Sizage(hs=2, ss=2, fs=None, ls=1), - '6D': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAD': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAD': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAD': Sizage(hs=4, ss=4, fs=None, ls=2), - '4E': Sizage(hs=2, ss=2, fs=None, ls=0), - '5E': Sizage(hs=2, ss=2, fs=None, ls=1), - '6E': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAE': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAE': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAE': Sizage(hs=4, ss=4, fs=None, ls=2), - } # Bards table maps first code char. converted to binary sextext of hard size, # hs. Used for ._bexfil. Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) - def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, - qb64b=None, qb64=None, qb2=None, strip=False): + # Sizes table maps from value of hs chars of code to Sizage namedtuple of + # (hs, ss, xs, fs, ls) where hs is hard size, ss is soft size, + # xs is extra size of soft, fs is full size, and ls is lead size of raw. + Sizes = { + 'A': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'B': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'C': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'D': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'E': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'F': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'G': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'H': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'I': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'J': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'K': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'L': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'M': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'N': Sizage(hs=1, ss=0, xs=0, fs=12, ls=0), + 'O': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'P': Sizage(hs=1, ss=0, xs=0, fs=124, ls=0), + 'Q': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'R': Sizage(hs=1, ss=0, xs=0, fs=8, ls=0), + 'S': Sizage(hs=1, ss=0, xs=0, fs=16, ls=0), + 'T': Sizage(hs=1, ss=0, xs=0, fs=20, ls=0), + 'U': Sizage(hs=1, ss=0, xs=0, fs=24, ls=0), + 'V': Sizage(hs=1, ss=0, xs=0, fs=4, ls=1), + 'W': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'X': Sizage(hs=1, ss=3, xs=0, fs=4, ls=0), + 'Y': Sizage(hs=1, ss=7, xs=0, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + '0A': Sizage(hs=2, ss=0, xs=0, fs=24, ls=0), + '0B': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0C': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0D': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0E': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0F': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0G': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0H': Sizage(hs=2, ss=0, xs=0, fs=8, ls=0), + '0I': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0J': Sizage(hs=2, ss=2, xs=1, fs=4, ls=0), + '0K': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0L': Sizage(hs=2, ss=6, xs=1, fs=8, ls=0), + '0M': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0N': Sizage(hs=2, ss=10, xs=1, fs=12, ls=0), + '0O': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '1AAA': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAC': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAD': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAE': Sizage(hs=4, ss=0, xs=0, fs=56, ls=0), + '1AAF': Sizage(hs=4, ss=4, xs=0, fs=8, ls=0), + '1AAG': Sizage(hs=4, ss=0, xs=0, fs=36, ls=0), + '1AAH': Sizage(hs=4, ss=0, xs=0, fs=100, ls=0), + '1AAI': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAJ': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAK': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAL': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAM': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAN': Sizage(hs=4, ss=8, xs=0, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, xs=1, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=2), + '4A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + } + + Codes = asdict(MtrDex) # map code name to code + Names = {val : key for key, val in Codes.items()} # invert map code to code name + Pad = '_' # B64 pad char for special codes with xtra size pre-padded soft values + + + def __init__(self, raw=None, code=MtrDex.Ed25519N, soft='', rize=None, + qb64b=None, qb64=None, qb2=None, strip=False, **kwa): """ Validate as fully qualified Parameters: - raw (bytes): unqualified crypto material usable for crypto operations + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. code (str): stable (hard) part of derivation code - rize (int): raw size in bytes when variable sized material else None - qb64b (bytes): fully qualified crypto material Base64 - qb64 (str, bytes): fully qualified crypto material Base64 - qb2 (bytes): fully qualified crypto material Base2 + soft (str | bytes): soft part exclusive of prepad for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (str | bytes | bytearray | memoryview | None): fully qualified + crypto material Base64. When str, encodes as utf-8. Strips when + bytearray and strip is True. + qb64 (str | bytes | bytearray | memoryview | None): fully qualified + crypto material Base64. When str, encodes as utf-8. Ignores strip + qb2 (bytes | bytearray | memoryview | None): fully qualified crypto + material Base2. Strips when bytearray and strip is True. strip (bool): True means strip (delete) matter from input stream bytearray after parsing qb64b or qb2. False means do not strip - Needs either (raw and code and optionally size and rsize) + Needs either (raw and code and optionally rsize) or qb64b or qb64 or qb2 Otherwise raises EmptyMaterialError - When raw and code and optional size and rsize provided - then validate that code is correct for length of raw, size, rsize - and assign .raw + When raw and code and optional rsize provided + then validate that code is correct for length of raw, rsize, + computed size from Sizes and assign .raw Else when qb64b or qb64 or qb2 provided extract and assign .raw and .code and .size and .rsize """ - size = None # variable raw binary size including leader in quadlets - if raw is not None: # raw provided + if hasattr(soft, "decode"): # make soft str + soft = soft.decode("utf-8") + + if raw is not None: # raw provided but may be empty if not code: raise EmptyMaterialError(f"Improper initialization need either " - f"(raw and code) or qb64b or qb64 or qb2.") + f"(raw not None and code) or " + f"(code and soft) or " + f"qb64b or qb64 or qb2.") if not isinstance(raw, (bytes, bytearray)): - raise TypeError(f"Not a bytes or bytearray, raw={raw}.") + raise TypeError(f"Not a bytes or bytearray {raw=}.") if code not in self.Sizes: - raise InvalidCodeError("Unsupported code={}.".format(code)) + raise InvalidCodeError(f"Unsupported {code=}.") - if code[0] in SmallVrzDex or code[0] in LargeVrzDex: # dynamic size - if rize: # use rsize to determin length of raw to extract + hs, ss, xs, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + + if fs is None: # variable sized assumes code[0] in SmallVrzDex or LargeVrzDex + # assumes xs must be 0 when variable sized + if rize: # use rsize to determine length of raw to extract if rize < 0: raise InvalidVarRawSizeError(f"Missing var raw size for " f"code={code}.") @@ -963,46 +920,83 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, ls = (3 - (rize % 3)) % 3 # calc actual lead (pad) size # raw binary size including leader in bytes size = (rize + ls) // 3 # calculate value of size in triplets + if code[0] in SmallVrzDex: # compute code with sizes - if size <= (64 ** 2 - 1): + if size <= (64 ** 2 - 1): # ss = 2 hs = 2 s = astuple(SmallVrzDex)[ls] code = f"{s}{code[1:hs]}" - elif size <= (64 ** 4 - 1): # make big version of code + ss = 2 + elif size <= (64 ** 4 - 1): # ss = 4 make big version of code hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{'A' * (hs - 2)}{code[1]}" + soft = intToB64(size, 4) + ss = 4 else: - raise InvalidVarRawSizeError(r"Unsupported raw size for " - f"code={code}.") + raise InvalidVarRawSizeError(f"Unsupported raw size for " + f"{code=}.") elif code[0] in LargeVrzDex: # compute code with sizes - if size <= (64 ** 4 - 1): + if size <= (64 ** 4 - 1): # ss = 4 hs = 4 s = astuple(LargeVrzDex)[ls] code = f"{s}{code[1:hs]}" + ss = 4 else: - raise InvalidVarRawSizeError(r"Unsupported raw size for " - f"code={code}.") + raise InvalidVarRawSizeError(f"Unsupported raw size for large " + f"{code=}. {size} <= {64 ** 4 - 1}") else: - raise InvalidVarRawSizeError(r"Unsupported variable raw size " - f"code={code}.") + raise InvalidVarRawSizeError(f"Unsupported variable raw size " + f"{code=}.") + soft = intToB64(size, ss) + + else: # fixed size but raw may be empty and/or special soft + rize = Matter._rawSize(code) # get raw size from Sizes for code + # if raw then ls may be nonzero + + if ss > 0: # special soft size, so soft must be provided + soft = soft[:ss-xs] # + if len(soft) != ss - xs: + raise SoftMaterialError(f"Not enough chars in {soft=} " + f"with {ss=} {xs=} for {code=}.") + + if not Reb64.match(soft.encode("utf-8")): + raise InvalidSoftError(f"Non Base64 chars in {soft=}.") + else: + soft = '' # must be empty when ss == 0 - else: - hs, ss, fs, ls = self.Sizes[code] # get sizes assumes ls consistent - if not fs: # invalid - raise InvalidVarSizeError(r"Unsupported variable size " - f"code={code}.") - rize = Matter._rawSize(code) raw = raw[:rize] # copy only exact size from raw stream if len(raw) != rize: # forbids shorter raise RawMaterialError(f"Not enougth raw bytes for code={code}" - f"expected {rize} got {len(raw)}.") + f"expected {rize=} got {len(raw)}.") - self._code = code # hard value part of code - self._size = size # soft value part of code in int + self._code = code # str hard part of full code + self._soft = soft # str soft part of full code exclusive of xs prepad, empty when ss=0 self._raw = bytes(raw) # crypto ops require bytes not bytearray + elif soft and code: # raw None so ls == 0 with fixed size and special + hs, ss, xs, fs, ls = self.Sizes[code] # assumes unit tests force valid sizes + if not fs: # variable sized code so can't be special soft + raise InvalidSoftError(f"Unsupported variable sized {code=} " + f" with {fs=} for special {soft=}.") + + if not ss > 0 or (fs == hs + ss and not ls == 0): # not special soft + raise InvalidSoftError("Invalid soft size={ss} or lead={ls} " + f" or {code=} {fs=} when special soft.") + + soft = soft[:ss-xs] + if len(soft) != ss - xs: + raise SoftMaterialError(f"Not enough chars in {soft=} " + f"with {ss=} {xs=} for {code=}.") + + if not Reb64.match(soft.encode("utf-8")): + raise InvalidSoftError(f"Non Base64 chars in {soft=}.") + + self._code = code # str hard part of code + self._soft = soft # str soft part of code, empty when ss=0 + self._raw = b'' # force raw empty when None given and special soft + elif qb64b is not None: self._exfil(qb64b) if strip: # assumes bytearray @@ -1018,7 +1012,10 @@ def __init__(self, raw=None, code=MtrDex.Ed25519N, rize=None, else: raise EmptyMaterialError(f"Improper initialization need either " - f"(raw and code) or qb64b or qb64 or qb2.") + f"(raw not None and code) or " + f"(code and soft) or " + f"qb64b or qb64 or qb2.") + @classmethod def _rawSize(cls, code): @@ -1027,12 +1024,14 @@ def _rawSize(cls, code): Parameters: code (str): derivation code Base64 """ - hs, ss, fs, ls = cls.Sizes[code] # get sizes + hs, ss, xs, fs, ls = cls.Sizes[code] # get sizes cs = hs + ss # both hard + soft code size if fs is None: raise InvalidCodeSizeError(f"Non-fixed raw size code {code}.") + # assumes .Sizes only has valid entries, cs % 4 != 3, and fs % 4 == 0 return (((fs - cs) * 3 // 4) - ls) + @classmethod def _leadSize(cls, code): """ @@ -1040,37 +1039,113 @@ def _leadSize(cls, code): Parameters: code (str): derivation code Base64 """ - _, _, _, ls = cls.Sizes[code] # get lead size from .Sizes table + _, _, _, _, ls = cls.Sizes[code] # get lead size from .Sizes table return ls - @property - def code(self): + @classmethod + def _xtraSize(cls, code): """ - Returns ._code which is the hard part only of full text code. - Some codes only have a hard part. Soft part is for variable sized matter. - Makes .code read only + Returns xtra size in bytes for a given code + Parameters: + code (str): derivation code Base64 + """ + _, _, xs, _, _ = cls.Sizes[code] # get lead size from .Sizes table + return xs + + + @classmethod + def _special(cls, code): + """ + Returns: + special (bool): True when code has special soft i.e. when + fs is not None and ss > 0 + False otherwise + + """ + hs, ss, xs, fs, ls = cls.Sizes[code] + + return (fs is not None and ss > 0) + + + @property + def code(self): + """ + Returns: + code (str): hard part only of full text code. + + Getter for ._code. Makes ._code read only + + Some codes only have a hard part. Soft part may be for variable sized + matter or for special codes that are code only (raw is empty) """ return self._code + @property - def both(self): + def name(self): + """ + Returns: + name (str): code name for self.code. Used for annotation for + primitives like Matter + + """ + return self.Names[self.code] + + + @property + def hard(self): + """ + Returns: + hard (str): hard part only of full text code. Alias for .code. + + """ + return self.code + + + @property + def soft(self): """ - Returns both hard and soft parts of full text code + Returns: + soft (str): soft part only of full text code. + + Getter for ._soft. Make ._soft read only """ - _, ss, _, _ = self.Sizes[self.code] - return (f"{self.code}{intToB64(self.size, l=ss)}") + return self._soft @property def size(self): """ - Returns ._size int or None if not variable sized matter - Makes .size read only + Returns: + size(int | None): Number of variably sized b64 quadlets/b2 triplets + in primitive when varibly sized + None when not variably sized when (fs!=None) + + Number of quadlets/triplets of chars/bytes of variable sized material or + None when not variably sized. - Number of triplets of bytes including lead bytes (quadlets of chars) - of variable sized material. Value of soft size, ss, part of full text code. + Converted qb64 value to int of soft ss portion of full text code + when variably sized primitive material (fs == None). """ - return self._size + return (b64ToInt(self.soft) if self.soft else None) + + + @property + def both(self): + """ + Returns: + both (str): hard + soft parts of full text code + """ + #_, ss, _, _ = self.Sizes[self.code] + + #if self.size is not None: + #return (f"{self.code}{intToB64(self.size, l=ss)}") + #else: + #return (f"{self.code}{self.soft}") + + _, _, xs, _, _ = self.Sizes[self.code] + + return (f"{self.code}{self.Pad * xs}{self.soft}") @property @@ -1080,12 +1155,13 @@ def fullSize(self): Fixed size codes returns fs from .Sizes Variable size codes where fs==None computes fs from .size and sizes """ - hs, ss, fs, _ = self.Sizes[self.code] # get sizes + hs, ss, _, fs, _ = self.Sizes[self.code] # get sizes if fs is None: # compute fs from ss characters in code fs = hs + ss + (self.size * 4) return fs + @property def raw(self): """ @@ -1094,6 +1170,7 @@ def raw(self): """ return self._raw + @property def qb64b(self): """ @@ -1103,6 +1180,7 @@ def qb64b(self): """ return self._infil() + @property def qb64(self): """ @@ -1112,6 +1190,7 @@ def qb64(self): """ return self.qb64b.decode("utf-8") + @property def qb2(self): """ @@ -1120,6 +1199,7 @@ def qb2(self): """ return self._binfil() + @property def transferable(self): """ @@ -1129,6 +1209,7 @@ def transferable(self): """ return (self.code not in NonTransDex) + @property def digestive(self): """ @@ -1149,107 +1230,141 @@ def prefixive(self): return (self.code in PreDex) + @property + def special(self): + """ + special (bool): True when self.code has special self.soft i.e. when + fs is not None and ss > 0 and fs = hs + ss and ls = 0 + i.e. (fs fixed and soft not empty and raw is empty and no lead) + False otherwise + """ + return self._special(self.code) + + @property + def composable(self): + """ + composable (bool): True when both .qb64b and .qb2 are 24 bit aligned and + round trip using encodeB64 and decodeB64. + False otherwise + """ + qb64b = self.qb64b + qb2 = self.qb2 + return (len(qb64b) % 4 == 0 and len(qb2) % 3 == 0 and + encodeB64(qb2) == qb64b and decodeB64(qb64b) == qb2) + + def _infil(self): """ - Returns bytes of fully qualified base64 characters - self.code + converted self.raw to Base64 with pad chars stripped + Create text domain representation + Returns: + primitive (bytes): fully qualified base64 characters. + """ + code = self.code # hard part of full code == codex value + both = self.both # code + soft, soft may be empty + raw = self.raw # bytes or bytearray, raw may be empty + rs = len(raw) # raw size + hs, ss, xs, fs, ls = self.Sizes[code] cs = hs + ss - fs = (size * 4) + cs - - """ - code = self.code # hard size codex value - size = self.size # size if variable length, None otherwise - raw = self.raw # bytes or bytearray - - ps = ((3 - (len(raw) % 3)) % 3) # pad size chars or lead size bytes - hs, ss, fs, ls = self.Sizes[code] - if not fs: # variable sized, compute code ss value from .size - cs = hs + ss # both hard + soft size - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - - if size < 0 or size > (64 ** ss - 1): - raise InvalidVarSizeError("Invalid size={} for code={}." - "".format(size, code)) - # both is hard code + size converted to ss B64 chars - both = f"{code}{intToB64(size, l=ss)}" - - if len(both) % 4 != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - # prepad, convert, and prepend - return (both.encode("utf-8") + encodeB64(bytes([0] * ls) + raw)) - - else: # fixed size so prepad but lead ls may not be zero - both = code - cs = len(both) - if (cs % 4) != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - # prepad, convert, and replace upfront - # when fixed and ls != 0 then cs % 4 is zero and ps==ls - # otherwise fixed and ls == 0 then cs % 4 == ps - return (both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[cs % 4:]) + # assumes unit tests on Matter.Sizes ensure valid size entries + + if cs != len(both): + InvalidCodeSizeError(f"Invalid full code={both} for sizes {hs=} and" + f" {ss=}.") + + if not fs: # variable sized + # Tests on .Sizes table must ensure ls in (0,1,2) and cs % 4 == 0 but + # can't know the variable size. So instance methods must ensure that + # (ls + rs) % 3 == 0 i.e. both full code (B64) and lead+raw (B2) + # are both 24 bit aligned. + # If so then should not need following check. + if (ls + rs) % 3 or cs % 4: + raise InvalidCodeSizeError(f"Invalid full code{both=} with " + f"variable raw size={rs} given " + f" {cs=}, {hs=}, {ss=}, {fs=}, and " + f"{ls=}.") + + # When ls+rs is 24 bit aligned then encodeB64 has no trailing + # pad chars that need to be stripped. So simply prepad raw with + # ls zero bytes and convert (encodeB64). + full = (both.encode("utf-8") + encodeB64(bytes([0] * ls) + raw)) + + else: # fixed size + ps = (3 - ((rs + ls) % 3)) % 3 # net pad size given raw with lead + # net pad size must equal both code size remainder so that primitive + # both + converted padded raw is fs long. Assumes ls in (0,1,2) and + # cs % 4 != 3, fs % 4 == 0. Sizes table test must ensure these properties. + # If so then should not need following check. + if ps != (cs % 4): # given cs % 4 != 3 then cs % 4 is pad size + raise InvalidCodeSizeError(f"Invalid full code{both=} with " + f"fixed raw size={rs} given " + f" {cs=}, {hs=}, {ss=}, {fs=}, and " + f"{ls=}.") + + # Predpad raw so we midpad the full primitive. Prepad with ps+ls + # zero bytes ensures encodeB64 of prepad+lead+raw has no trailing + # pad characters. Finally skip first ps == cs % 4 of the converted + # characters to ensure that when full code is prepended, the full + # primitive size is fs but midpad bits are zeros. + full = (both.encode("utf-8") + encodeB64(bytes([0] * (ps + ls)) + raw)[ps:]) + + if (len(full) % 4) or (fs and len(full) != fs): + raise InvalidCodeSizeError(f"Invalid full size given code{both=} " + f" with raw size={rs}, {cs=}, {hs=}, " + f"{ss=}, {xs=} {fs=}, and {ls=}.") + + return full def _binfil(self): """ + Create binary domain representation + Returns bytes of fully qualified base2 bytes, that is .qb2 self.code converted to Base2 + self.raw left shifted with pad bits equivalent of Base64 decode of .qb64 into .qb2 """ - code = self.code # codex value - size = self.size # optional size if variable length - raw = self.raw # bytes or bytearray + code = self.code # hard part of full code == codex value + both = self.both # code + soft, soft may be empty + raw = self.raw # bytes or bytearray may be empty - hs, ss, fs, ls = self.Sizes[code] + hs, ss, xs, fs, ls = self.Sizes[code] cs = hs + ss - - if not fs: # compute both and fs from size - if cs % 4: - raise InvalidCodeSizeError("Whole code size not multiple of 4 for " - "variable length material. cs={}.".format(cs)) - - if size < 0 or size > (64 ** ss - 1): - raise InvalidVarSizeError("Invalid size={} for code={}." - "".format(size, code)) - # both is hard code + converted index - both = f"{code}{intToB64(size, l=ss)}" - fs = hs + ss + (size * 4) - else: - both = code - - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(code))) - + # assumes unit tests on Matter.Sizes ensure valid size entries n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code # convert code both to right align b2 int then left shift in pad bits # then convert to bytes bcode = (b64ToInt(both) << (2 * (cs % 4))).to_bytes(n, 'big') - full = bcode + bytes([0] * ls) + raw + full = bcode + bytes([0] * ls) + raw # includes lead bytes + bfs = len(full) + if not fs: # compute fs + fs = hs + ss + (len(raw) + ls) * 4 // 3 # hs + ss + (size * 4) if bfs % 3 or (bfs * 4 // 3) != fs: # invalid size - raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") - + raise InvalidCodeSizeError(f"Invalid full code={both} for raw size" + f"={len(raw)}.") return full def _exfil(self, qb64b): """ - Extracts self.code and self.raw from qualified base64 bytes qb64b + Extracts self.code and self.raw from qualified base64 qb64b of type + str or bytes or bytearray or memoryview + + Detects if str and converts to bytes + + Parameters: + qb64b (str | bytes | bytearray | memoryview): fully qualified base64 from stream - cs = hs + ss - fs = (size * 4) + cs """ if not qb64b: # empty need more bytes raise ShortageError("Empty material.") first = qb64b[:1] # extract first char code selector + if isinstance(first, memoryview): + first = bytes(first) if hasattr(first, "decode"): - first = first.decode("utf-8") + first = first.decode() # converts bytes/bytearray to str if first not in self.Hards: if first[0] == '-': raise UnexpectedCountCodeError("Unexpected count code start" @@ -1265,67 +1380,65 @@ def _exfil(self, qb64b): raise ShortageError(f"Need {hs - len(qb64b)} more characters.") hard = qb64b[:hs] # extract hard code + if isinstance(hard, memoryview): + hard = bytes(hard) if hasattr(hard, "decode"): - hard = hard.decode("utf-8") # converts bytes/bytearray to str + hard = hard.decode() # converts bytes/bytearray to str if hard not in self.Sizes: raise UnexpectedCodeError(f"Unsupported code ={hard}.") - hs, ss, fs, ls = self.Sizes[hard] # assumes hs in both tables match + hs, ss, xs, fs, ls = self.Sizes[hard] # assumes hs in both tables match cs = hs + ss # both hs and ss - size = None - if not fs: # compute fs from size chars in ss part of code - if cs % 4: - raise ValidationError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - size = qb64b[hs:hs + ss] # extract size chars - if hasattr(size, "decode"): - size = size.decode("utf-8") - size = b64ToInt(size) # compute int size - fs = (size * 4) + cs - - # assumes that unit tests on Matter and MatterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and ss == 0 and not fs % 4 and hs > 0 and fs >= hs + ss - # unless fs is None + # assumes that unit tests on Matter .Sizes .Hards and .Bards ensure that + # these are well formed. + # when fs is None then ss > 0 otherwise fs > hs + ss when ss > 0 + + + # extract soft chars including xtra, empty when ss==0 and xs == 0 + # assumes that when ss == 0 then xs must be 0 + soft = qb64b[hs:hs+ss] + if isinstance(soft, memoryview): + soft = bytes(soft) + if hasattr(soft, "decode"): + soft = soft.decode() # converts bytes/bytearray to str + xtra = soft[:xs] # extract xtra if any from front of soft + soft = soft[xs:] # strip xtra from soft + if xtra != f"{self.Pad * xs}": + raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") + + if not fs: # compute fs from soft from ss part which provides size B64 + # compute variable size as int may have value 0 + fs = (b64ToInt(soft) * 4) + cs if len(qb64b) < fs: # need more bytes raise ShortageError(f"Need {fs - len(qb64b)} more chars.") qb64b = qb64b[:fs] # fully qualified primitive code plus material + if isinstance(qb64b, memoryview): + qb64b = bytes(qb64b) if hasattr(qb64b, "encode"): # only convert extracted chars from stream - qb64b = qb64b.encode("utf-8") - - # check for non-zeroed pad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero - paw = decodeB64(base) # decode base to leave prepadded raw - pi = (int.from_bytes(paw[:ps], "big")) # prepad as int - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed prepad bits = " - f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") - raw = paw[ps:] # strip off ps prepad paw bytes - - else: # not ps. IF not ps THEN may or may not be ls (lead) - base = qb64b[cs:] # strip off code leaving lead chars if any and value - # decode lead chars + val leaving lead bytes + raw bytes - # then strip off ls lead bytes leaving raw - paw = decodeB64(base) # decode base to leave prepadded paw bytes - li = int.from_bytes(paw[:ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") - raw = paw[ls:] # paw is bytes so raw is bytes + qb64b = qb64b.encode() # converts str to bytes + + # check for non-zeroed pad bits and/or lead bytes + # net prepad ps == cs % 4 (remainer). Assumes ps != 3 i.e ps in (0,1,2) + # To ensure number of prepad bytes and prepad chars are same. + # need net prepad chars ps to invert using decodeB64 of lead + raw + + ps = cs % 4 # net prepad bytes to ensure 24 bit align when encodeB64 + base = ps * b'A' + qb64b[cs:] # prepad ps 'A's to B64 of (lead + raw) + paw = decodeB64(base) # now should have ps + ls leading sextexts of zeros + raw = paw[ps+ls:] # remove prepad midpat bytes to invert back to raw + # ensure midpad bytes are zero + pi = int.from_bytes(paw[:ps+ls], "big") + if pi != 0: + raise ConversionError(f"Nonzero midpad bytes=0x{pi:0{(ps + ls) * 2}x}.") if len(raw) != ((len(qb64b) - cs) * 3 // 4) - ls: # exact lengths raise ConversionError(f"Improperly qualified material = {qb64b}") - self._code = hard # hard only - self._size = size - self._raw = raw # ensure bytes so immutable and for crypto ops + self._code = hard # hard only str + self._soft = soft # soft only str + self._raw = raw # ensure bytes for crypto ops, may be empty def _bexfil(self, qb2): @@ -1333,7 +1446,7 @@ def _bexfil(self, qb2): Extracts self.code and self.raw from qualified base2 qb2 Parameters: - qb2 (bytes | bytearray): fully qualified base2 from stream + qb2 (bytes | bytearray | memoryview): fully qualified base2 from stream """ if not qb2: # empty need more bytes raise ShortageError("Empty material, Need more bytes.") @@ -1358,70 +1471,76 @@ def _bexfil(self, qb2): if hard not in self.Sizes: raise UnexpectedCodeError(f"Unsupported code ={hard}.") - hs, ss, fs, ls = self.Sizes[hard] + hs, ss, xs, fs, ls = self.Sizes[hard] cs = hs + ss # both hs and ss + # assumes that unit tests on Matter .Sizes .Hards and .Bards ensure that + # these are well formed. + # when fs is None then ss > 0 otherwise fs > hs + ss when ss > 0 + bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets - size = None - if not fs: # compute fs from size chars in ss part of code - if cs % 4: - raise ValidationError("Whole code size not multiple of 4 for " - "variable length material. cs={}.".format(cs)) + if len(qb2) < bcs: # need more bytes + raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) + both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code + + # extract soft chars including xtra, empty when ss==0 and xs == 0 + # assumes that when ss == 0 then xs must be 0 + soft = both[hs:hs+ss] # get soft may be empty + xtra = soft[:xs] # extract xtra if any from front of soft + soft = soft[xs:] # strip xtra from soft + if xtra != f"{self.Pad * xs}": + raise UnexpectedCodeError(f"Invalid prepad xtra ={xtra}.") + + if not fs: # compute fs from size chars in ss part of code if len(qb2) < bcs: # need more bytes raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - size = b64ToInt(both[hs:hs + ss]) # get size - fs = (size * 4) + cs - - # assumes that unit tests on Matter and MatterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and ss == 0 and not fs % 4 and hs > 0 and - # (fs >= hs + ss if fs is not None else True) + # compute size as int from soft part given by ss B64 chars + fs = (b64ToInt(soft) * 4) + cs # compute fs bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets if len(qb2) < bfs: # need more bytes raise ShortageError("Need {} more bytes.".format(bfs - len(qb2))) qb2 = qb2[:bfs] # extract qb2 fully qualified primitive code plus material - # check for non-zeroed prepad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - # convert last byte of code bytes in which are pad bits to int - pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed pad bits = " - f"{pi & (2 ** pbs - 1 ):>08b} in 0x{pi:02x}.") - else: # not ps. IF not ps THEN may or may not be ls (lead) - li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:02x}.") - raw = qb2[(bcs + ls):] # strip code and leader bytes from qb2 to get raw + # check for nonzero trailing full code mid pad bits + ps = cs % 4 # full code (both) net pad size for 24 bit alignment + pbs = 2 * ps # mid pad bits = 2 per net pad + # get pad bits in last byte of full code + pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) # convert byte to int + pi = pi & (2 ** pbs - 1 ) # mask with 1's in pad bit locations + if pi: # not zero so raise error + raise ConversionError(f"Nonzero code mid pad bits=0b{pi:0{pbs}b}.") + + # check nonzero leading mid pad lead bytes in lead + raw + li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int + if li: # midpad lead bytes must be zero + raise ConversionError(f"Nonzero lead midpad bytes=0x{li:0{ls*2}x}.") + + # strip code and leader bytes from qb2 to get raw + raw = qb2[(bcs + ls):] # may be empty if len(raw) != (len(qb2) - bcs - ls): # exact lengths raise ConversionError(r"Improperly qualified material = {qb2}") - self._code = hard - self._size = size - self._raw = bytes(raw) # ensure bytes so immutable and crypto operations + self._code = hard # hard only + self._soft = soft # soft only may be empty + self._raw = bytes(raw) # ensure bytes for crypto ops may be empty class Seqner(Matter): """ - Seqner is subclass of Matter, cryptographic material, for ordinal numbers - such as sequence numbers or first seen ordering numbers. - Seqner provides fully qualified format for ordinals (sequence numbers etc) - when provided as attached cryptographic material elements. + Seqner is subclass of Matter, cryptographic material, for fully qualified + fixed serialization sized ordinal numbers such as sequence numbers or + first seen numbers. - Useful when parsing attached receipt groupings with sn from stream or database + The serialization is forced to a fixed size (single code) so that it may be + used for lexocographically ordered namespaces such as database indices. + That code is MtrDex.Salt_128 - Uses default initialization code = CryTwoDex.Salt_128 - Raises error on init if code not CryTwoDex.Salt_128 + Default initialization code = MtrDex.Salt_128 + Raises error on init if code is not MtrDex.Salt_128 Attributes: @@ -1447,10 +1566,8 @@ class Seqner(Matter): ._infil is method to compute fully qualified Base64 from .raw and .code ._exfil is method to extract .code and .raw from fully qualified Base64 - Methods: - """ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, @@ -1466,16 +1583,32 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, Parameters: - sn is int sequence number or some form of ordinal number - snh is hex string of sequence number + sn (int | str | None): some form of ordinal number int or hex str + snh (str | None): hex string of ordinal number """ if raw is None and qb64b is None and qb64 is None and qb2 is None: - if sn is None: - if snh is None: - sn = 0 - else: - sn = int(snh, 16) + try: + if sn is None: + if snh is None or snh == '': + sn = 0 + else: + sn = int(snh, 16) + + else: # sn is not None but so may be hex str + if isinstance(sn, str): # is it a hex str + if sn == '': + sn = 0 + else: + sn = int(sn, 16) + except ValueError as ex: + raise InvalidValueError(f"Not whole number={sn} .") from ex + + if not isinstance(sn, int) or sn < 0: + raise InvalidValueError(f"Not whole number={sn}.") + + if sn > MaxON: # too big for ordinal 256 ** 16 - 1 + raise ValidationError(f"Non-ordinal {sn} exceeds {MaxON}.") raw = sn.to_bytes(Matter._rawSize(MtrDex.Salt_128), 'big') @@ -1486,6 +1619,7 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, raise ValidationError("Invalid code = {} for Seqner." "".format(self.code)) + @property def sn(self): """ @@ -1503,6 +1637,7 @@ def snh(self): return f"{self.sn:x}" # "{:x}".format(self.sn) + class Number(Matter): """ Number is subclass of Matter, cryptographic material, for ordinal counting @@ -1537,8 +1672,14 @@ class Number(Matter): Properties: num (int): int representation of number humh (str): hex string representation of number with no leading zeros + sn (int): alias for num + snh (str): alias for numh + huge (str): qb64 of num but with code NumDex.Huge so 24 char compatible + with fixed size seq num for lexicographic lmdb key space positive (bool): True if .num > 0, False otherwise. Because .num must be - non-negative, .positive == False means .num == 0 + non-negative, .positive == False means .num == 0 + inceptive (bool): True means .num == 0 False otherwise. + Hidden: _code (str): value for .code property @@ -1555,13 +1696,19 @@ class Number(Matter): Methods: """ + Codes = asdict(NumDex) # map code name to code + Names = {val : key for key, val in Codes.items()} # invert map code to code name + + def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, - code=NumDex.Short, num=None, numh=None, **kwa): + code=None, num=None, numh=None, **kwa): """ Inherited Parameters: (see Matter) raw (bytes): unqualified crypto material usable for crypto operations - code (str): stable (hard) part of derivation code + code (str | None): stable (hard) part of derivation code. + None means pick code based on value of num or numh + otherwise raise error rize (int): raw size in bytes when variable sized material else None qb64b (bytes): fully qualified crypto material Base64 qb64 (str, bytes): fully qualified crypto material Base64 @@ -1583,8 +1730,6 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if numh is None or numh == '': num = 0 else: - #if len(numh) > 32: - #raise InvalidValueError(f"Hex numh={numh} str too long.") num = int(numh, 16) else: # handle case where num is hex str' @@ -1592,33 +1737,44 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if num == '': num = 0 else: - #if len(num) > 32: - #raise InvalidValueError(f"Hex num={num} str too long.") num = int(num, 16) except ValueError as ex: - raise InvalidValueError(f"Invalid whole number={num} .") from ex + raise InvalidValueError(f"Not whole number={num} .") from ex - if not isinstance(num, int) or num < 0: - raise InvalidValueError(f"Invalid whole number={num}.") + if code is None: # dynamically size code + if not isinstance(num, int) or num < 0: + raise InvalidValueError(f"Not whole number={num}.") - if num <= (256 ** 2 - 1): # make short version of code - code = NumDex.Short + if num <= (256 ** 2 - 1): # make short version of code + code = NumDex.Short - elif num <= (256 ** 4 - 1): # make long version of code - code = code = NumDex.Long + elif num <= (256 ** 5 - 1): # make tall version of code + code = code = NumDex.Tall - elif num <= (256 ** 8 - 1): # make big version of code - code = code = NumDex.Big + elif num <= (256 ** 8 - 1): # make big version of code + code = code = NumDex.Big - elif num <= (256 ** 16 - 1): # make huge version of code - code = code = NumDex.Huge + elif num <= (256 ** 11 - 1): # make large version of code + code = code = NumDex.Large - else: - raise InvalidValueError(f"Invalid num = {num}, too large to encode.") + elif num <= (256 ** 14 - 1): # make great version of code + code = code = NumDex.Great + + elif num <= (256 ** 17 - 1): # make vast version of code + code = code = NumDex.Vast + + else: + raise InvalidValueError(f"Invalid num = {num}, too large to encode.") # default to_bytes parameter signed is False. If negative raises # OverflowError: can't convert negative int to unsigned - raw = num.to_bytes(Matter._rawSize(code), 'big') # big endian unsigned + try: + raw = num.to_bytes(Matter._rawSize(code), 'big') # big endian unsigned + except Exception as ex: + raise InvalidValueError(f"Not convertable to bytes {num=}.") from ex + + if len(raw) > Matter._rawSize(code): + raise InvalidValueError(f"To big {num=} for {code=}.") super(Number, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, code=code, **kwa) @@ -1627,6 +1783,44 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, raise ValidationError(f"Invalid code = {self.code} for Number.") + def validate(self, inceptive=None): + """ + Returns: + self (Number): + + Raises: + ValidationError: when .num is invalid ordinal such as + sequence number or first seen number etc. + + Parameters: + inceptive(bool): raise ValidationError whan .num invalid + None means exception when .num < 0 + True means exception when .num != 0 + False means exception when .num < 1 + + """ + num = self.num + + if num > MaxON: # too big for ordinal 256 ** 16 - 1 + raise ValidationError(f"Non-ordinal {num} exceeds {MaxON}.") + + if inceptive is not None: + if inceptive: + if num != 0: + raise ValidationError(f"Nonzero num = {num} non-inceptive" + f" ordinal.") + else: + if num < 1: + raise ValidationError(f"Non-positive num = {num} not " + f"non-inceptive ordinal.") + else: + if num < 0: + raise ValidationError(f"Negative num = {num} non-ordinal.") + + return self + + + @property def num(self): """ @@ -1662,6 +1856,20 @@ def snh(self): return self.numh + @property + def huge(self): + """Provides number value as qb64 but with code NumDex.huge. This is the + same as Seqner.qb64. Raises error if too big. + + Returns: + huge (str): qb64 of num coded as NumDex.Huge + """ + num = self.num + if num > MaxON: # too big for ordinal 256 ** 16 - 1 + raise InvalidValueError(f"Non-ordinal {num} exceeds {MaxON}.") + + return Number(num=num, code=NumDex.Huge).qb64 + @property def positive(self): @@ -1672,16 +1880,17 @@ def positive(self): """ return True if self.num > 0 else False + @property def inceptive(self): """ Returns True if .num == 0 False otherwise. - Because valid number .num must be non-negative, positive False means - that .num is zero. + """ return True if self.num == 0 else False + class Dater(Matter): """ Dater is subclass of Matter, cryptographic material, for RFC-3339 profile of @@ -1774,8 +1983,7 @@ def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, if raw is None and qb64b is None and qb64 is None and qb2 is None: if dts is None: # defaults to now dts = helping.nowIso8601() - # if len(dts) != 32: - # raise ValueError("Invalid length of date time string") + if hasattr(dts, "decode"): dts = dts.decode("utf-8") qb64 = MtrDex.DateTime + dts.translate(self.ToB64) @@ -1811,934 +2019,666 @@ def datetime(self): return helping.fromIso8601(self.dts) -class Bexter(Matter): +class Tagger(Matter): """ - Bexter is subclass of Matter, cryptographic material, for variable length - strings that only contain Base64 URL safe characters, i.e. Base64 text (bext). - When created using the 'bext' paramaeter, the encoded matter in qb64 format - in the text domain is more compact than would be the case if the string were - passed in as raw bytes. The text is used as is to form the value part of the - qb64 version not including the leader. + Tagger is subclass of Matter, cryptographic material, for compact special + fixed size primitive with non-empty soft part and empty raw part. - Due to ambiguity that arises from pre-padding bext whose length is a multiple of - three with one or more 'A' chars. Any bext that starts with an 'A' and whose length - is either a multiple of 3 or 4 may not round trip. Bext with a leading 'A' - whose length is a multiple of four may have the leading 'A' stripped when - round tripping. + Tagger provides a more compact representation of small Base64 values in + as soft part of code rather than would be obtained by by using a small raw + part whose ASCII representation is converted to Base64. - Bexter(bext='ABBB').bext == 'BBB' - Bexter(bext='BBB').bext == 'BBB' - Bexter(bext='ABBB').qb64 == '4AABABBB' == Bexter(bext='BBB').qb64 + Attributes: - To avoid this problem, only use for applications of base 64 strings that - never start with 'A' + Inherited Properties: (See Matter) + code (str): hard part of derivation code to indicate cypher suite + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip - Examples: base64 text strings: + Properties: + tag (str): B64 .soft portion of code but without prepad - bext = "" - qb64 = '4AAA' - bext = "-" - qb64 = '6AABAAA-' + Inherited Hidden: (See Matter) + _code (str): value for .code property + _soft (str): soft value of full code + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) - bext = "-A" - qb64 = '5AABAA-A' - bext = "-A-" - qb64 = '4AABA-A-' + Hidden: - bext = "-A-B" - qb64 = '4AAB-A-B' + Methods: - Example uses: - CESR encoded paths for nested SADs and SAIDs - CESR encoded fractionally weighted threshold expressions + """ - Attributes: - Inherited Properties: (See Matter) - .pad is int number of pad chars given raw + def __init__(self, tag='', soft='', code=None, **kwa): + """ + Inherited Parameters: (see Matter) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. + code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .index is int count of attached crypto material by context (receipts) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - .transferable is Boolean, True when transferable derivation code False otherwise + Parameters: + tag (str | bytes): Base64 automatic sets code given size of tag - Properties: - .text is the Base64 text value, .qb64 with text code and leader removed. + """ + if tag: + if hasattr(tag, "encode"): # make tag bytes for regex + tag = tag.encode("utf-8") - Hidden: - ._pad is method to compute .pad property - ._code is str value for .code property - ._raw is bytes value for .raw property - ._index is int value for .index property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 + if not Reb64.match(tag): + raise InvalidSoftError(f"Non Base64 chars in {tag=}.") - Methods: + code = self._codify(tag=tag) + soft = tag - """ - def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, - code=MtrDex.StrB64_L0, bext=None, **kwa): - """ - Inherited Parameters: (see Matter) - raw is bytes of unqualified crypto material usable for crypto operations - qb64b is bytes of fully qualified crypto material - qb64 is str or bytes of fully qualified crypto material - qb2 is bytes of fully qualified crypto material - code is str of derivation code - index is int of count of attached receipts for CryCntDex codes + super(Tagger, self).__init__(soft=soft, code=code, **kwa) - Parameters: - bext is the variable sized Base64 text string - """ - if raw is None and qb64b is None and qb64 is None and qb2 is None: - if bext is None: - raise EmptyMaterialError("Missing bext string.") - if hasattr(bext, "encode"): - bext = bext.encode("utf-8") - if not Reb64.match(bext): - raise ValueError("Invalid Base64.") - raw = self._rawify(bext) + if (not self._special(self.code)) or self.code not in TagDex: + raise InvalidCodeError(f"Invalid code={self.code} for Tagger.") - super(Bexter, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, - code=code, **kwa) - if self.code not in BexDex: - raise ValidationError("Invalid code = {} for Bexter." - "".format(self.code)) - def _rawify(self, bext): - """Returns raw value equivalent of Base64 text. - Suitable for variable sized matter + @staticmethod + def _codify(tag): + """Returns code for tag when tag is appropriately sized Base64 Parameters: - text (bytes): Base64 bytes + tag (str | bytes): Base64 value + + Returns: + code (str): derivation code for tag + """ - ts = len(bext) % 4 # bext size mod 4 - ws = (4 - ts) % 4 # pre conv wad size in chars - ls = (3 - ts) % 3 # post conv lead size in bytes - base = b'A' * ws + bext # pre pad with wad of zeros in Base64 == 'A' - raw = decodeB64(base)[ls:] # convert and remove leader - return raw # raw binary equivalent of text + # TagDex tags appear in order of size 1 to 10, at indices 0 to 9 + codes = astuple(TagDex) + l = len(tag) + if l < 1 or l > len(codes): + raise InvalidSoftError(f"Invalid {tag=} size {l=}, empty or oversized.") + return codes[l-1] # return code at index = len - 1 + + @property - def bext(self): - """ - Property bext: Base64 text value portion of qualified b64 str - Returns the value portion of .qb64 with text code and leader removed + def tag(self): + """Returns: + tag (str): B64 primitive without prepad (alias of self.soft) + """ - _, _, _, ls = self.Sizes[self.code] - bext = encodeB64(bytes([0] * ls) + self.raw) - ws = 0 - if ls == 0 and bext: - if bext[0] == ord(b'A'): # strip leading 'A' zero pad - ws = 1 - else: - ws = (ls + 1) % 4 - return bext.decode('utf-8')[ws:] + return self.soft -class Pather(Bexter): +class Ilker(Tagger): """ - Pather is a subclass of Bexter that provides SAD Path language specific functionality - for variable length strings that only contain Base64 URL safe characters. Pather allows - the specification of SAD Paths as a list of field components which will be converted to the - Base64 URL safe character representation. + Ilker is subclass of Tagger, cryptographic material, for formatted + message types (ilks) in Base64. Leverages Tagger support compact special + fixed size primitives with non-empty soft part and empty raw part. - Additionally, Pather provides .rawify for extracting and serializing the content targeted by - .path for a SAD, represented as an instance of Serder. Pather enforces Base64 URL character - safety by leveraging the fact that SADs must have static field ordering. Any field label can - be replaced by its field ordinal to allow for path specification and traversal for any field - labels that contain non-Base64 URL safe characters. + Ilker provides a more compact representation than would be obtained by + converting the raw ASCII representation to Base64. + Attributes: - Examples: strings: - path = [] - text = "-" - qb64 = '6AABAAA-' + Inherited Properties: (See Tagger) + code (str): hard part of derivation code to indicate cypher suite + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip + tag (str): B64 primitive without prepad (strips prepad from soft) - path = ["A"] - text = "-A" - qb64 = '5AABAA-A' - path = ["A", "B"] - text = "-A-B" - qb64 = '4AAB-A-B' + Properties: + ilk (str): message type from Ilks of Ilkage + + Inherited Hidden: (See Tagger) + _code (str): value for .code property + _soft (str): soft value of full code + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) + + Hidden: - path = ["A", 1, "B", 3] - text = "-A-1-B-3" - qb64 = '4AAC-A-1-B-3' + + Methods: """ - def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, bext=None, - code=MtrDex.StrB64_L0, path=None, **kwa): + + def __init__(self, qb64b=None, qb64=None, qb2=None, tag='', ilk='', **kwa): """ - Inherited Parameters: (see Bexter) - raw is bytes of unqualified crypto material usable for crypto operations - qb64b is bytes of fully qualified crypto material - qb64 is str or bytes of fully qualified crypto material - qb2 is bytes of fully qualified crypto material - code is str of derivation code - index is int of count of attached receipts for CryCntDex codes - bext is the variable sized Base64 text string + Inherited Parameters: (see Tagger) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. + code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip + tag (str | bytes): Base64 plain. Prepad is added as needed. Parameters: - path (list): array of path field components + ilk (str): message type from Ilks of Ilkage + """ + if not (qb64b or qb64 or qb2): + if ilk: + tag = ilk - if raw is None and bext is None and qb64b is None and qb64 is None and qb2 is None: - if path is None: - raise EmptyMaterialError("Missing path list.") - bext = self._bextify(path) + super(Ilker, self).__init__(qb64b=qb64b, qb64=qb64, qb2=qb2, tag=tag, **kwa) - super(Pather, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, bext=bext, - code=code, **kwa) + if self.code not in (MtrDex.Tag3, ): + raise InvalidCodeError(f"Invalid code={self.code} for Ilker " + f"{self.ilk=}.") + if self.ilk not in Ilks: + raise InvalidSoftError(f"Ivalid ilk={self.ilk} for Ilker.") - @property - def path(self): - """ Path property is an array of path elements - Path property is an array of path elements. Empty path represents the top level. - Returns: - list: array of field specs of the path + @property + def ilk(self): + """Returns: + tag (str): B64 primitive without prepad (strips prepad from soft) + + Alias for self.tag """ - if not self.bext.startswith("-"): - raise Exception("invalid SAD ptr") + return self.tag - path = self.bext.strip("-").split("-") - return path if path[0] != '' else [] - def root(self, root): - """ Returns a new Pather anchored at new root +class Traitor(Tagger): + """ + Traitor is subclass of Tagger, cryptographic material, for formatted + configuration traits for key events in Base64. Leverages Tagger support of + compact special fixed size primitives with non-empty soft part and empty raw part. - Returns a new Pather anchoring this path at the new root specified by root. + Traitor provides a more compact representation than would be obtained by + converting the raw ASCII representation to Base64. - Args: - root(Pather): the new root to apply to this path + Attributes: - Returns: - Pather: new path anchored at root - """ - return Pather(path=root.path + self.path) + Inherited Properties: (See Tagger) + code (str): hard part of derivation code to indicate cypher suite + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip + tag (str): B64 primitive without prepad (strips prepad from soft) - def strip(self, root): - """ Returns a new Pather with root stipped off the front if it exists - Returns a new Pather with root stripped off the front + Properties: + trait (str): configuration trait B64 from TraitDex - Args: - root(Pather): the new root to apply to this path + Inherited Hidden: (See Tagger) + _code (str): value for .code property + _soft (str): soft value of full code + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) - Returns: - Pather: new path anchored at root - """ - if len(root.path) > len(self.path): - return Pather(path=self.path) + Hidden: - path = list(self.path) - try: - for i in root.path: - path.remove(i) - except ValueError: - return Pather(path=self.path) - return Pather(path=path) + Methods: - def startswith(self, path): - """ Returns True if path is the root of self + """ - Parameters: - path (Pather): the path to check against self - Returns: - bool: True if path is the root of self + def __init__(self, qb64b=None, qb64=None, qb2=None, tag='', trait='', **kwa): + """ + Inherited Parameters: (see Tagger) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. + code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip + tag (str | bytes): Base64 plain. Prepad is added as needed. + + Parameters: + trait (str): configuration trait B64 from TraitDex """ + if not (qb64b or qb64 or qb2): + if trait: + tag = trait - return self.bext.startswith(path.bext) - def resolve(self, sad): - """ Recurses thru value following ptr + super(Traitor, self).__init__(qb64b=qb64b, qb64=qb64, qb2=qb2, tag=tag, **kwa) - Parameters: - sad(dict or list): the next component - Returns: - Value at the end of the path - """ - return self._resolve(sad, self.path) + if self.trait not in TraitDex: + raise InvalidSoftError(f"Invalid trait={self.trait} for Traitor.") - def tail(self, serder): - """ Recurses thru value following .path and returns terminal value - Finds the value at this path and applies the version string rules of the serder - to serialize the value at ptr. + @property + def trait(self): + """Returns: + trait (str): B64 primitive without prepad (strips prepad from soft) - Parameters: - serder(Serder): the versioned dict to in which to resolve .path + Alias for self.tag - Returns: - bytes: Value at the end of the path """ - val = self.resolve(sad=serder.sad) - if isinstance(val, str): - saider = Saider(qb64=val) - return saider.qb64b - elif isinstance(val, dict): - return dumps(val, serder.kind) - elif isinstance(val, list): - return dumps(val, serder.kind) - else: - raise ValueError("Bad tail value at {} of {}" - .format(self.path, serder.ked)) + return self.tag - @staticmethod - def _bextify(path): - """ Returns Base64 text delimited equivalent of path components - Suitable for variable sized matter +# Versage namedtuple +# proto (str): protocol element of Protocols +# vrsn (Versionage): instance protocol version namedtuple (major, minor) ints +# vrsn (Versionage | None): instance genus version namedtuple (major, minor) ints +Versage = namedtuple("Versage", "proto vrsn gvrsn", defaults=(None, )) - Parameters: - path (list): array of path field components - Returns: - str: textual representation of SAD path +class Verser(Tagger): + """ + Verser is subclass of Tagger, cryptographic material, for formatted + version primitives in Base64. Leverages Tagger support compact special + fixed size primitives with non-empty soft part and empty raw part. - """ - vath = [] # valid path components - for p in path: - if hasattr(p, "decode"): - p = p.decode("utf-8") + Verser provides a more compact representation than would be obtained by + converting the raw ASCII representation to Base64. - elif isinstance(p, int): - p = str(p) + Attributes: - if not Reb64.match(p.encode("utf-8")): - raise ValueError(f"Non Base64 path component = {p}.") + Inherited Properties: (See Tagger) + code (str): hard part of derivation code to indicate cypher suite + hard (str): hard part of derivation code. alias for code + soft (str): soft part of derivation code fs any. + Empty when ss = 0. + both (str): hard + soft parts of full text code + size (int | None): Number of quadlets/triplets of chars/bytes including + lead bytes of variable sized material (fs = None). + Converted value of the soft part (of len ss) of full + derivation code. + Otherwise None when not variably sized (fs != None) + fullSize (int): full size of primitive + raw (bytes): crypto material only. Not derivation code or lead bytes. + qb64 (str): Base64 fully qualified with derivation code + crypto mat + qb64b (bytes): Base64 fully qualified with derivation code + crypto mat + qb2 (bytes): binary with derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + prefixive (bool): True means identifier prefix derivation code False otherwise + special (bool): True when soft is special raw is empty and fixed size + composable (bool): True when .qb64b and .qb2 are 24 bit aligned and round trip + tag (str): B64 primitive without prepad (strips prepad from soft) - vath.append(p) - return ("-" + "-".join(vath)) + Properties: + versage (Versage): named tuple of (proto, vrsn, gvrsn) - def _resolve(self, val, ptr): - """ Recurses thru value following ptr + Inherited Hidden: (See Tagger) + _code (str): value for .code property + _soft (str): soft value of full code + _raw (bytes): value for .raw property + _rawSize(): + _leadSize(): + _special(): + _infil(): creates qb64b from .raw and .code (fully qualified Base64) + _binfil(): creates qb2 from .raw and .code (fully qualified Base2) + _exfil(): extracts .code and .raw from qb64b (fully qualified Base64) + _bexfil(): extracts .code and .raw from qb2 (fully qualified Base2) - Parameters: - val(Optional(dict,list)): the next component - ptr(list): list of path components + Hidden: - Returns: - Value at the end of the chain - """ - if len(ptr) == 0: - return val + Methods: - idx = ptr.pop(0) + """ - if isinstance(val, dict): - if idx.isdigit(): - i = int(idx) - keys = list(val) - if i >= len(keys): - raise KeyError(f"invalid dict pointer index {i} for keys {keys}") + def __init__(self, qb64b=None, qb64=None, qb2=None, versage=None, + proto=Protocols.keri, vrsn=Vrsn_2_0, gvrsn=None, tag='', **kwa): + """ + Inherited Parameters: (see Tagger) + raw (bytes | bytearray | None): unqualified crypto material usable + for crypto operations. + code (str): stable (hard) part of derivation code + soft (str | bytes): soft part for special codes + rize (int | None): raw size in bytes when variable sized material not + including lead bytes if any + Otherwise None + qb64b (bytes | None): fully qualified crypto material Base64 + qb64 (str | bytes | None): fully qualified crypto material Base64 + qb2 (bytes | None): fully qualified crypto material Base2 + strip (bool): True means strip (delete) matter from input stream + bytearray after parsing qb64b or qb2. False means do not strip + tag (str | bytes): Base64 plain. Prepad is added as needed. - cur = val[list(val)[i]] - elif idx == "": - return val - else: - cur = val[idx] + Parameters: + versage (Versage | None): namedtuple of (proto, vrsn, gvrsn) + proto (str | None): protocol from Protocols + vrsn (Versionage | None): instance protocol version. + namedtuple (major, minor) of ints + gvrsn (Versionage | None): instance genus version. + namedtuple (major, minor) of ints - elif isinstance(val, list): - i = int(idx) - if i >= len(val): - raise KeyError(f"invalid array pointer index {i} for array {val}") + """ + if not (qb64b or qb64 or qb2): + if versage: + proto, vrsn, gvrsn = versage - cur = val[i] + tag = proto + self.verToB64(vrsn) - else: - raise KeyError("invalid traversal type") + if gvrsn: + tag += self.verToB64(gvrsn) - return self._resolve(cur, ptr) + super(Verser, self).__init__(qb64b=qb64b, qb64=qb64, qb2=qb2, tag=tag, **kwa) + if self.code not in (MtrDex.Tag7, MtrDex.Tag10, ): + raise InvalidCodeError(f"Invalid code={self.code} for " + f"Verser={self.tag}.") -class Verfer(Matter): - """ - Verfer is Matter subclass with method to verify signature of serialization - using the .raw as verifier key and .code for signature cipher suite. + @property + def versage(self): + """Returns: + versage (Versage): named tuple of (proto, vrsn, gvrsn) - See Matter for inherited attributes and properties: + """ + gvrsn = None + proto = self.tag[:4] + vrsn = self.b64ToVer(self.tag[4:7]) + gvrsn = self.b64ToVer(self.tag[7:10]) if len(self.tag) == 10 else None - Attributes: + return Versage(proto=proto, vrsn=vrsn, gvrsn=gvrsn) - Properties: - Methods: - verify: verifies signature + @staticmethod + def verToB64(version=None, *, text="", major=0, minor=0): + """ Converts version to Base64 representation - """ - - def __init__(self, **kwa): - """ - Assign verification cipher suite function to ._verify - - """ - super(Verfer, self).__init__(**kwa) - - if self.code in [MtrDex.Ed25519N, MtrDex.Ed25519]: - self._verify = self._ed25519 - elif self.code in [MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256r1]: - self._verify = self._secp256r1 - elif self.code in [MtrDex.ECDSA_256k1N, MtrDex.ECDSA_256k1]: - self._verify = self._secp256k1 - else: - raise ValueError("Unsupported code = {} for verifier.".format(self.code)) + Returns: + verB64 (str): - def verify(self, sig, ser): - """ - Returns True if bytes signature sig verifies on bytes serialization ser - using .raw as verifier public key for ._verify cipher suite determined - by .code + Example: + Verser.verToB64(verstr = "1.0")) Parameters: - sig is bytes signature - ser is bytes serialization - """ - return (self._verify(sig=sig, ser=ser, key=self.raw)) + version (Versionage): instange of namedtuple + Versionage(major=major,minor=minor) + text (str): text format of version as dotted decimal "major.minor" + major (int): When version is None and verstr is empty then use major minor + range [0, 63] for one Base64 character + minor (int): When version is None and verstr is empty then use major minor + range [0, 4095] for two Base64 characters - @staticmethod - def _ed25519(sig, ser, key): """ - Returns True if verified False otherwise - Verify Ed25519 sig on ser using key + if version: + major = version.major + minor = version.minor - Parameters: - sig is bytes signature - ser is bytes serialization - key is bytes public key - """ - try: # verify returns None if valid else raises ValueError - pysodium.crypto_sign_verify_detached(sig, ser, key) - except Exception as ex: - return False + elif text: + splits = text.split(".", maxsplit=2) + splits = [(int(s) if s else 0) for s in splits] + parts = [major, minor] + for i in range(2-len(splits),0, -1): # append missing minor and/or major + splits.append(parts[-i]) + major = splits[0] + minor = splits[1] - return True + if major < 0 or major > 63 or minor < 0 or minor > 4095: + raise ValueError(f"Out of bounds version = {major}.{minor}.") - @staticmethod - def _secp256r1(sig, ser, key): - """ - Returns True if verified False otherwise - Verify secp256r1 sig on ser using key + return (f"{intToB64(major)}{intToB64(minor, l=2)}") - Parameters: - sig is bytes signature - ser is bytes serialization - key is bytes public key - """ - verkey = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), key) - r = int.from_bytes(sig[:32], "big") - s = int.from_bytes(sig[32:], "big") - der = utils.encode_dss_signature(r, s) - try: - verkey.verify(der, ser, ec.ECDSA(hashes.SHA256())) - return True - except exceptions.InvalidSignature: - return False @staticmethod - def _secp256k1(sig, ser, key): - """ - Returns True if verified False otherwise - Verify secp256k1 sig on ser using key - - Parameters: - sig is bytes signature - ser is bytes serialization - key is bytes public key - """ - verkey = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256K1(), key) - r = int.from_bytes(sig[:32], "big") - s = int.from_bytes(sig[32:], "big") - der = utils.encode_dss_signature(r, s) - try: - verkey.verify(der, ser, ec.ECDSA(hashes.SHA256())) - return True - except exceptions.InvalidSignature: - return False - - -class Cigar(Matter): - """ - Cigar is Matter subclass holding a nonindexed signature with verfer property. - From Matter .raw is signature and .code is signature cipher suite - Adds .verfer property to hold Verfer instance of associated verifier public key - Verfer's .raw as verifier key and .code is verifier cipher suite. - - See Matter for inherited attributes and properties: - - Attributes: - - Properties: (Inherited) - .code is str derivation code to indicate cypher suite - .size is size (int): number of quadlets when variable sized material besides - full derivation code else None - .raw is bytes crypto material only without code - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - .transferable is Boolean, True when transferable derivation code False otherwise - .digestive is Boolean, True when digest derivation code False otherwise - - Properties: - .verfer is verfer of public key used to verify signature + def b64ToVer(b64, *, texted=False): + """ Converts Base64 representation of version to Versionage or + text dotted decimal format - Hidden: - ._code is str value for .code property - ._size is int value for .size property - ._raw is bytes value for .raw property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 + default is Versionage - Methods: + Returns: + version (Versionage | str): - Hidden: - ._pad is method to compute .pad property - ._code is str value for .code property - ._raw is bytes value for .raw property - ._index is int value for .index property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 + Example: + .b64ToVer("BAA")) - """ + Parameters: + b64 (str): base64 string of three characters Mmm for Major minor + texted (bool): return text format dotted decimal string - def __init__(self, verfer=None, **kwa): - """ - Assign verfer to ._verfer attribute """ - super(Cigar, self).__init__(**kwa) - self._verfer = verfer + if not Reb64.match(b64.encode("utf-8")): + raise ValueError("Invalid Base64.") - @property - def verfer(self): - """ - Property verfer: - Returns Verfer instance - Assumes ._verfer is correctly assigned - """ - return self._verfer + if texted: + return ".".join([f"{b64ToInt(b64[0])}", f"{b64ToInt(b64[1:3])}"]) - @verfer.setter - def verfer(self, verfer): - """ verfer property setter """ - self._verfer = verfer + return Versionage(major=b64ToInt(b64[0]), minor=b64ToInt(b64[1:3])) -class Signer(Matter): +class Texter(Matter): """ - Signer is Matter subclass with method to create signature of serialization - using: - .raw as signing (private) key seed, - .code as cipher suite for signing - .verfer whose property .raw is public key for signing. - - If not provided .verfer is generated from private key seed using .code - as cipher suite for creating key-pair. - + Texter is subclass of Matter, cryptographic material, for variable length + text strings as bytes not unicode. Unicode strings converted to bytes. - See Matter for inherited attributes and properties: Attributes: - Properties: (inherited) - code (str): hard part of derivation code to indicate cypher suite - both (int): hard and soft parts of full text code - size (int): Number of triplets of bytes including lead bytes - (quadlets of chars) of variable sized material. Value of soft size, - ss, part of full text code. - Otherwise None. - rize (int): number of bytes of raw material not including - lead bytes - raw (bytes): private signing key crypto material only without code - qb64 (str): private signing key Base64 fully qualified with - derivation code + crypto mat - qb64b (bytes): private signing keyBase64 fully qualified with - derivation code + crypto mat - qb2 (bytes): private signing key binary with - derivation code + crypto material - transferable (bool): True means transferable derivation code False otherwise - digestive (bool): True means digest derivation code False otherwise + Inherited Properties: (See Matter) Properties: + .text is bytes value with CESR code and leader removed. + .uext is str value with CESR code and leader removed unicode of .text - .verfer is Verfer object instance of public key derived from private key - seed which is .raw + Inherited Hidden Properties: (See Matter) Methods: - sign: create signature + + Codes: + Bytes_L0: str = '4B' # Byte String lead size 0 + Bytes_L1: str = '5B' # Byte String lead size 1 + Bytes_L2: str = '6B' # Byte String lead size 2 + Bytes_Big_L0: str = '7AAB' # Byte String big lead size 0 + Bytes_Big_L1: str = '8AAB' # Byte String big lead size 1 + Bytes_Big_L2: str = '9AAB' # Byte String big lead size 2 """ - def __init__(self, raw=None, code=MtrDex.Ed25519_Seed, transferable=True, **kwa): + def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, + code=MtrDex.Bytes_L0, text=None, **kwa): """ - Assign signing cipher suite function to ._sign + Inherited Parameters: (see Matter) + raw is bytes of unqualified crypto material usable for crypto operations + qb64b is bytes of fully qualified crypto material + qb64 is str or bytes of fully qualified crypto material + qb2 is bytes of fully qualified crypto material + code is str of derivation code + index is int of count of attached receipts for CryCntDex codes - Parameters: See Matter for inherted parameters - raw is bytes crypto material seed or private key - code is derivation code - transferable is Boolean True means make verifier code transferable - False make non-transferable + Parameters: + text is the variable sized text string as either bytes or str """ - try: - super(Signer, self).__init__(raw=raw, code=code, **kwa) - except EmptyMaterialError as ex: - if code == MtrDex.Ed25519_Seed: - raw = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - super(Signer, self).__init__(raw=raw, code=code, **kwa) - elif code == MtrDex.ECDSA_256r1_Seed: - raw = pysodium.randombytes(ECDSA_256r1_SEEDBYTES) - super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) - elif code == MtrDex.ECDSA_256k1_Seed: - raw = pysodium.randombytes(ECDSA_256k1_SEEDBYTES) - super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) + if raw is None and qb64b is None and qb64 is None and qb2 is None: + if text is None: + raise EmptyMaterialError("Missing text string.") + if hasattr(text, "encode"): + text = text.encode("utf-8") + raw = text - else: - raise ValueError("Unsupported signer code = {}.".format(code)) - - if self.code == MtrDex.Ed25519_Seed: - self._sign = self._ed25519 - verkey, sigkey = pysodium.crypto_sign_seed_keypair(self.raw) - verfer = Verfer(raw=verkey, - code=MtrDex.Ed25519 if transferable - else MtrDex.Ed25519N) - elif self.code == MtrDex.ECDSA_256r1_Seed: - self._sign = self._secp256r1 - d = int.from_bytes(self.raw, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256R1()) - verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) - verfer = Verfer(raw=verkey, - code=MtrDex.ECDSA_256r1 if transferable - else MtrDex.ECDSA_256r1N) - elif self.code == MtrDex.ECDSA_256k1_Seed: - self._sign = self._secp256k1 - d = int.from_bytes(self.raw, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256K1()) - verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) - verfer = Verfer(raw=verkey, - code=MtrDex.ECDSA_256k1 if transferable - else MtrDex.ECDSA_256k1N) - else: - raise ValueError("Unsupported signer code = {}.".format(self.code)) + super(Texter, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, + code=code, **kwa) + if self.code not in TexDex: + raise ValidationError("Invalid code = {} for Texter." + "".format(self.code)) - self._verfer = verfer @property - def verfer(self): + def text(self): """ - Property verfer: - Returns Verfer instance - Assumes ._verfer is correctly assigned + Property text: raw as str """ - return self._verfer + return self.raw.decode('utf-8') - def sign(self, ser, index=None, only=False, ondex=None, **kwa): - """ - Returns either Cigar or Siger (indexed) instance of cryptographic - signature material on bytes serialization ser - If index is None - return Cigar instance - Else - return Siger instance +class Bexter(Matter): + """ + Bexter is subclass of Matter, cryptographic material, for variable length + strings that only contain Base64 URL safe characters, i.e. Base64 text (bext). + When created using the 'bext' paramaeter, the encoded matter in qb64 format + in the text domain is more compact than would be the case if the string were + passed in as raw bytes. The text is used as is to form the value part of the + qb64 version not including the leader. - Parameters: - ser (bytes): serialization to be signed - index (int): main index of associated verifier key in event keys - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next + Due to ambiguity that arises from pre-padding bext whose length is a multiple of + three with one or more 'A' chars. Any bext that starts with an 'A' and whose length + is either a multiple of 3 or 4 may not round trip. Bext with a leading 'A' + whose length is a multiple of four may have the leading 'A' stripped when + round tripping. - """ - return (self._sign(ser=ser, - seed=self.raw, - verfer=self.verfer, - index=index, - only=only, - ondex=ondex, - **kwa)) + Bexter(bext='ABBB').bext == 'BBB' + Bexter(bext='BBB').bext == 'BBB' + Bexter(bext='ABBB').qb64 == '4AABABBB' == Bexter(bext='BBB').qb64 - @staticmethod - def _ed25519(ser, seed, verfer, index, only=False, ondex=None, **kwa): - """ - Returns signature as either Cigar or Siger instance as appropriate for - Ed25519 digital signatures given index and ondex values + To avoid this problem, only use for applications of base 64 strings that + never start with 'A' - The seed's code determins the crypto key-pair algorithm and signing suite - The signature type, Cigar or Siger, and when indexed the Siger code - may be completely determined by the seed and index values (index, ondex) - by assuming that the index values are intentional. - Without the seed code its more difficult for Siger to - determine when for the Indexer code value should be changed from the - than the provided value with respect to provided but incompatible index - values versus error conditions. + Examples: base64 text strings: - Parameters: - ser (bytes): serialization to be signed - seed (bytes): raw binary seed (private key) - verfer (Verfer): instance. verfer.raw is public key - index (int |None): main index offset into list such as current signing - None means return non-indexed Cigar - Not None means return indexed Siger with Indexer code derived - from index, conly, and ondex values - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next - """ - # compute raw signature sig using seed on serialization ser - sig = pysodium.crypto_sign_detached(ser, seed + verfer.raw) - - if index is None: # Must be Cigar i.e. non-indexed signature - return Cigar(raw=sig, code=MtrDex.Ed25519_Sig, verfer=verfer) - else: # Must be Siger i.e. indexed signature - # should add Indexer class method to get ms main index size for given code - if only: # only main index ondex not used - ondex = None - if index <= 63: # (64 ** ms - 1) where ms is main index size - code = IdrDex.Ed25519_Crt_Sig # use small current only - else: - code = IdrDex.Ed25519_Big_Crt_Sig # use big current only - else: # both - if ondex == None: - ondex = index # enable default to be same - if ondex == index and index <= 63: # both same and small - code = IdrDex.Ed25519_Sig # use small both same - else: # otherwise big or both not same so use big both - code = IdrDex.Ed25519_Big_Sig # use use big both - - return Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) + bext = "" + qb64 = '4AAA' - @staticmethod - def _secp256r1(ser, seed, verfer, index, only=False, ondex=None, **kwa): - """ - Returns signature as either Cigar or Siger instance as appropriate for - Ed25519 digital signatures given index and ondex values + bext = "-" + qb64 = '6AABAAA-' - The seed's code determins the crypto key-pair algorithm and signing suite - The signature type, Cigar or Siger, and when indexed the Siger code - may be completely determined by the seed and index values (index, ondex) - by assuming that the index values are intentional. - Without the seed code its more difficult for Siger to - determine when for the Indexer code value should be changed from the - than the provided value with respect to provided but incompatible index - values versus error conditions. + bext = "-A" + qb64 = '5AABAA-A' - Parameters: - ser (bytes): serialization to be signed - seed (bytes): raw binary seed (private key) - verfer (Verfer): instance. verfer.raw is public key - index (int |None): main index offset into list such as current signing - None means return non-indexed Cigar - Not None means return indexed Siger with Indexer code derived - from index, conly, and ondex values - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next - """ - # compute raw signature sig using seed on serialization ser - d = int.from_bytes(seed, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256R1()) - der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) - (r, s) = utils.decode_dss_signature(der) - sig = bytearray(r.to_bytes(32, "big")) - sig.extend(s.to_bytes(32, "big")) - - if index is None: # Must be Cigar i.e. non-indexed signature - return Cigar(raw=sig, code=MtrDex.ECDSA_256r1_Sig, verfer=verfer) - else: # Must be Siger i.e. indexed signature - # should add Indexer class method to get ms main index size for given code - if only: # only main index ondex not used - ondex = None - if index <= 63: # (64 ** ms - 1) where ms is main index size - code = IdrDex.ECDSA_256r1_Crt_Sig # use small current only - else: - code = IdrDex.ECDSA_256r1_Big_Crt_Sig # use big current only - else: # both - if ondex == None: - ondex = index # enable default to be same - if ondex == index and index <= 63: # both same and small - code = IdrDex.ECDSA_256r1_Sig # use small both same - else: # otherwise big or both not same so use big both - code = IdrDex.ECDSA_256r1_Big_Sig # use use big both - - return Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) + bext = "-A-" + qb64 = '4AABA-A-' - @staticmethod - def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): - """ - Returns signature as either Cigar or Siger instance as appropriate for - secp256k1 digital signatures given index and ondex values + bext = "-A-B" + qb64 = '4AAB-A-B' - The seed's code determins the crypto key-pair algorithm and signing suite - The signature type, Cigar or Siger, and when indexed the Siger code - may be completely determined by the seed and index values (index, ondex) - by assuming that the index values are intentional. - Without the seed code its more difficult for Siger to - determine when for the Indexer code value should be changed from the - than the provided value with respect to provided but incompatible index - values versus error conditions. - Parameters: - ser (bytes): serialization to be signed - seed (bytes): raw binary seed (private key) - verfer (Verfer): instance. verfer.raw is public key - index (int |None): main index offset into list such as current signing - None means return non-indexed Cigar - Not None means return indexed Siger with Indexer code derived - from index, conly, and ondex values - only (bool): True means main index only list, ondex ignored - False means both index lists (default), ondex used - ondex (int | None): other index offset into list such as prior next - """ - # compute raw signature sig using seed on serialization ser - d = int.from_bytes(seed, byteorder="big") - sigkey = ec.derive_private_key(d, ec.SECP256K1()) - der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) - (r, s) = utils.decode_dss_signature(der) - sig = bytearray(r.to_bytes(32, "big")) - sig.extend(s.to_bytes(32, "big")) - - if index is None: # Must be Cigar i.e. non-indexed signature - return Cigar(raw=sig, code=MtrDex.ECDSA_256k1_Sig, verfer=verfer) - else: # Must be Siger i.e. indexed signature - # should add Indexer class method to get ms main index size for given code - if only: # only main index ondex not used - ondex = None - if index <= 63: # (64 ** ms - 1) where ms is main index size - code = IdrDex.ECDSA_256k1_Crt_Sig # use small current only - else: - code = IdrDex.ECDSA_256k1_Big_Crt_Sig # use big current only - else: # both - if ondex == None: - ondex = index # enable default to be same - if ondex == index and index <= 63: # both same and small - code = IdrDex.ECDSA_256k1_Sig # use small both same - else: # otherwise big or both not same so use big both - code = IdrDex.ECDSA_256k1_Big_Sig # use use big both - - return Siger(raw=sig, - code=code, - index=index, - ondex=ondex, - verfer=verfer,) - - # def derive_index_code(code, index, only=False, ondex=None, **kwa): - # # should add Indexer class method to get ms main index size for given code - # if only: # only main index ondex not used - # ondex = None - # if index <= 63: # (64 ** ms - 1) where ms is main index size, use small current only - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Crt_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Crt_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Crt_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - # else: # use big current only - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Big_Crt_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Big_Crt_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Big_Crt_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - # else: # both - # if ondex == None: - # ondex = index # enable default to be same - # if ondex == index and index <= 63: # both same and small so use small both same - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - # else: # otherwise big or both not same so use big both - # if code == MtrDex.Ed25519_Seed: - # indxSigCode = IdrDex.Ed25519_Big_Sig - # elif code == MtrDex.ECDSA_256r1_Seed: - # indxSigCode = IdrDex.ECDSA_256r1_Big_Sig - # elif code == MtrDex.ECDSA_256k1_Seed: - # indxSigCode = IdrDex.ECDSA_256k1_Big_Sig - # else: - # raise ValueError("Unsupported signer code = {}.".format(code)) - - # return (indxSigCode, ondex) - -class Salter(Matter): - """ - Salter is Matter subclass to maintain random salt for secrets (private keys) - Its .raw is random salt, .code as cipher suite for salt + Example uses: + CESR encoded paths for nested SADs and SAIDs + CESR encoded fractionally weighted threshold expressions + Attributes: - .level is str security level code. Provides default level - Inherited Properties - .pad is int number of pad chars given raw - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .index is int count of attached crypto material by context (receipts) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - .transferable is Boolean, True when transferable derivation code False otherwise + Inherited Properties: (See Matter) Properties: + .bext is the Base64 text value, .qb64 with text code and leader removed. + + Inherited Hidden Properties: (See Matter) Methods: + ._rawify(self, bext) - Hidden: - ._pad is method to compute .pad property - ._code is str value for .code property - ._raw is bytes value for .raw property - ._index is int value for .index property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 + Codes: + StrB64_L0: str = '4A' # String Base64 Only Leader Size 0 + StrB64_L1: str = '5A' # String Base64 Only Leader Size 1 + StrB64_L2: str = '6A' # String Base64 Only Leader Size 2 + StrB64_Big_L0: str = '7AAA' # String Base64 Only Big Leader Size 0 + StrB64_Big_L1: str = '8AAA' # String Base64 Only Big Leader Size 1 + StrB64_Big_L2: str = '9AAA' # String Base64 Only Big Leader Size 2 """ - Tier = Tiers.low - def __init__(self, raw=None, code=MtrDex.Salt_128, tier=None, **kwa): + def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, + code=MtrDex.StrB64_L0, bext=None, **kwa): """ - Initialize salter's raw and code - - Inherited Parameters: + Inherited Parameters: (see Matter) raw is bytes of unqualified crypto material usable for crypto operations qb64b is bytes of fully qualified crypto material qb64 is str or bytes of fully qualified crypto material @@ -2747,610 +2687,525 @@ def __init__(self, raw=None, code=MtrDex.Salt_128, tier=None, **kwa): index is int of count of attached receipts for CryCntDex codes Parameters: - + bext is the variable sized Base64 text string """ - try: - super(Salter, self).__init__(raw=raw, code=code, **kwa) - except EmptyMaterialError as ex: - if code == MtrDex.Salt_128: - raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - super(Salter, self).__init__(raw=raw, code=code, **kwa) - else: - raise ValueError("Unsupported salter code = {}.".format(code)) - - if self.code not in (MtrDex.Salt_128,): - raise ValueError("Unsupported salter code = {}.".format(self.code)) - - self.tier = tier if tier is not None else self.Tier - - def stretch(self, *, size=32, path="", tier=None, temp=False): - """ - Returns (bytes): raw binary seed (secret) derived from path and .raw - and stretched to size given by code using argon2d stretching algorithm. + if raw is None and qb64b is None and qb64 is None and qb2 is None: + if bext is None: + raise EmptyMaterialError("Missing bext string.") + if hasattr(bext, "encode"): + bext = bext.encode("utf-8") # convert to bytes + if not Reb64.match(bext): + raise ValueError("Invalid Base64.") + raw = self._rawify(bext) # convert bytes to raw with padding + + super(Bexter, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, + code=code, **kwa) + if self.code not in BexDex: + raise ValidationError("Invalid code = {} for Bexter." + "".format(self.code)) + + @staticmethod + def _rawify(bext): + """Returns raw value equivalent of Base64 text. + Suitable for variable sized matter Parameters: - size (int): number of bytes in stretched seed - path (str): unique chars used in derivation of seed (secret) - tier (str): value from Tierage for security level of stretch - temp is Boolean, True means use quick method to stretch salt - for testing only, Otherwise use time set by tier to stretch + text (bytes): Base64 bytes """ - tier = tier if tier is not None else self.tier + ts = len(bext) % 4 # bext size mod 4 + ws = (4 - ts) % 4 # pre conv wad size in chars + ls = (3 - ts) % 3 # post conv lead size in bytes + base = b'A' * ws + bext # pre pad with wad of zeros in Base64 == 'A' + raw = decodeB64(base)[ls:] # convert and remove leader + return raw # raw binary equivalent of text - if temp: - opslimit = 1 # pysodium.crypto_pwhash_OPSLIMIT_MIN - memlimit = 8192 # pysodium.crypto_pwhash_MEMLIMIT_MIN + @classmethod + def _derawify(cls, raw, code): + """Returns decoded raw as B64 str aka bext value + + Returns: + bext (str): decoded raw as B64 str aka bext value + """ + _, _, _, _, ls = cls.Sizes[code] + bext = encodeB64(bytes([0] * ls) + raw) + ws = 0 + if ls == 0 and bext: + if bext[0] == ord(b'A'): # strip leading 'A' zero pad + ws = 1 else: - if tier == Tiers.low: - opslimit = 2 # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE - memlimit = 67108864 # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE - elif tier == Tiers.med: - opslimit = 3 # pysodium.crypto_pwhash_OPSLIMIT_MODERATE - memlimit = 268435456 # pysodium.crypto_pwhash_MEMLIMIT_MODERATE - elif tier == Tiers.high: - opslimit = 4 # pysodium.crypto_pwhash_OPSLIMIT_SENSITIVE - memlimit = 1073741824 # pysodium.crypto_pwhash_MEMLIMIT_SENSITIVE - else: - raise ValueError("Unsupported security tier = {}.".format(tier)) + ws = (ls + 1) % 4 + return bext.decode('utf-8')[ws:] - # stretch algorithm is argon2id - seed = pysodium.crypto_pwhash(outlen=size, - passwd=path, - salt=self.raw, - opslimit=opslimit, - memlimit=memlimit, - alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) - return (seed) - def signer(self, *, code=MtrDex.Ed25519_Seed, transferable=True, path="", - tier=None, temp=False): + @property + def bext(self): """ - Returns Signer instance whose .raw secret is derived from path and - salter's .raw and stretched to size given by code. The signers public key - for its .verfer is derived from code and transferable. - - Parameters: - code is str code of secret crypto suite - transferable is Boolean, True means use transferace code for public key - path is str of unique chars used in derivation of secret seed for signer - tier is str Tierage security level - temp is Boolean, True means use quick method to stretch salt - for testing only, Otherwise use more time to stretch + Property bext: Base64 text value portion of qualified b64 str + Returns the value portion of .qb64 with text code and leader removed """ - seed = self.stretch(size=Matter._rawSize(code), path=path, tier=tier, - temp=temp) - - return (Signer(raw=seed, code=code, transferable=transferable)) + return self._derawify(raw=self.raw, code=self.code) - def signers(self, count=1, start=0, path="", **kwa): - """ - Returns list of count number of Signer instances with unique derivation - path made from path prefix and suffix of start plus offset for each count - value from 0 to count - 1. +class Pather(Bexter): + """ + Pather is a subclass of Bexter that provides SAD Path language specific functionality + for variable length strings that only contain Base64 URL safe characters. Pather allows + the specification of SAD Paths as a list of field components which will be converted to the + Base64 URL safe character representation. - See .signer for parameters used to create each signer. + Additionally, Pather provides .rawify for extracting and serializing the content targeted by + .path for a SAD, represented as an instance of Serder. Pather enforces Base64 URL character + safety by leveraging the fact that SADs must have static field ordering. Any field label can + be replaced by its field ordinal to allow for path specification and traversal for any field + labels that contain non-Base64 URL safe characters. - """ - return [self.signer(path=f"{path}{i + start:x}", **kwa) for i in range(count)] + Examples: strings: + path = [] + text = "-" + qb64 = '6AABAAA-' -class Cipher(Matter): - """ - Cipher is Matter subclass holding a cipher text of a secret that may be - either a secret seed (private key) or secret salt with appropriate CESR code - to indicate which kind (which indicates size). The cipher text is created - with assymetric encryption using an unrelated (public, private) - encryption/decryption key pair. The public key is used for encryption the - private key for decryption. The default is to use X25519 sealed box encryption. + path = ["A"] + text = "-A" + qb64 = '5AABAA-A' - The Cipher instances .raw is the raw binary encrypted cipher text and its - .code indicates what type of secret has been encrypted. The cipher suite used - for the encryption/decryption is implied by the context where the cipher is - used. + path = ["A", "B"] + text = "-A-B" + qb64 = '4AAB-A-B' - See Matter for inherited attributes and properties + path = ["A", 1, "B", 3] + text = "-A-1-B-3" + qb64 = '4AAC-A-1-B-3' """ - def __init__(self, raw=None, code=None, **kwa): + def __init__(self, raw=None, qb64b=None, qb64=None, qb2=None, bext=None, + code=MtrDex.StrB64_L0, path=None, **kwa): """ - Parmeters: - raw (Union[bytes, str]): cipher text - code (str): cipher suite + Inherited Parameters: (see Bexter) + raw is bytes of unqualified crypto material usable for crypto operations + qb64b is bytes of fully qualified crypto material + qb64 is str or bytes of fully qualified crypto material + qb2 is bytes of fully qualified crypto material + code is str of derivation code + index is int of count of attached receipts for CryCntDex codes + bext is the variable sized Base64 text string + + Parameters: + path (list): array of path field components """ - if raw is not None and code is None: - if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): - code = MtrDex.X25519_Cipher_Salt - elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): - code = MtrDex.X25519_Cipher_Seed - if hasattr(raw, "encode"): - raw = raw.encode("utf-8") # ensure bytes not str + if raw is None and bext is None and qb64b is None and qb64 is None and qb2 is None: + if path is None: + raise EmptyMaterialError("Missing path list.") - super(Cipher, self).__init__(raw=raw, code=code, **kwa) + bext = self._bextify(path) - if self.code not in (MtrDex.X25519_Cipher_Salt, MtrDex.X25519_Cipher_Seed): - raise ValueError("Unsupported cipher code = {}.".format(self.code)) + super(Pather, self).__init__(raw=raw, qb64b=qb64b, qb64=qb64, qb2=qb2, bext=bext, + code=code, **kwa) - def decrypt(self, prikey=None, seed=None): - """ - Returns plain text as Matter instance (Signer or Salter) of cryptographic - cipher text material given by .raw. Encrypted plain text is fully - qualified (qb64) so derivaton code of plain text preserved through - encryption/decryption round trip. + @property + def path(self): + """ Path property is an array of path elements - Uses either decryption key given by prikey or derives prikey from - signing key derived from private seed. + Path property is an array of path elements. Empty path represents the top level. - Parameters: - prikey (Union[bytes, str]): qb64b or qb64 serialization of private - decryption key - seed (Union[bytes, str]): qb64b or qb64 serialization of private - signing key seed used to derive private decryption key - """ - decrypter = Decrypter(qb64b=prikey, seed=seed) - return decrypter.decrypt(ser=self.qb64b) + Returns: + list: array of field specs of the path + """ + if not self.bext.startswith("-"): + raise Exception("invalid SAD ptr") -class Encrypter(Matter): - """ - Encrypter is Matter subclass with method to create a cipher text of a - fully qualified (qb64) private key/seed where private key/seed is the plain - text. Encrypter uses assymetric (public, private) key encryption of a - serialization (plain text). Using its .raw as the encrypting (public) key and - its .code to indicate the cipher suite for the encryption operation. - - For example .code == MtrDex.X25519 indicates that X25519 sealed box - encyrption is used. The encryption key may be derived from an Ed25519 - signing public key that associated with a nontransferable or basic derivation - self certifying identifier. This allows use of the self certifying identifier - to track or manage the encryption/decryption key pair. And could be used to - provide additional authentication operations for using the - encryption/decryption key pair. Support for this is provided at init time - with the verkey parameter which allows deriving the encryption public key from - the fully qualified verkey (signature verification key). + path = self.bext.strip("-").split("-") + return path if path[0] != '' else [] - See Matter for inherited attributes and properties: + def root(self, root): + """ Returns a new Pather anchored at new root - Methods: - encrypt: returns cipher text + Returns a new Pather anchoring this path at the new root specified by root. - """ + Args: + root(Pather): the new root to apply to this path - def __init__(self, raw=None, code=MtrDex.X25519, verkey=None, **kwa): + Returns: + Pather: new path anchored at root """ - Assign encrypting cipher suite function to ._encrypt + return Pather(path=root.path + self.path) - Parameters: See Matter for inherted parameters such as qb64, qb64b - raw (bytes): public encryption key - qb64b (bytes): fully qualified public encryption key - qb64 (str): fully qualified public encryption key - code (str): derivation code for public encryption key - verkey (Union[bytes, str]): qb64b or qb64 of verkey used to derive raw - """ - if not raw and verkey: - verfer = Verfer(qb64b=verkey) - if verfer.code not in (MtrDex.Ed25519N, MtrDex.Ed25519): - raise ValueError("Unsupported verkey derivation code = {}." - "".format(verfer.code)) - # convert signing public key to encryption public key - raw = pysodium.crypto_sign_pk_to_box_pk(verfer.raw) + def strip(self, root): + """ Returns a new Pather with root stipped off the front if it exists - super(Encrypter, self).__init__(raw=raw, code=code, **kwa) + Returns a new Pather with root stripped off the front - if self.code == MtrDex.X25519: - self._encrypt = self._x25519 - else: - raise ValueError("Unsupported encrypter code = {}.".format(self.code)) + Args: + root(Pather): the new root to apply to this path - def verifySeed(self, seed): - """ Returns: - Boolean: True means private signing key seed corresponds to public - signing key verkey used to derive encrypter's .raw public - encryption key. + Pather: new path anchored at root + """ + if len(root.path) > len(self.path): + return Pather(path=self.path) + + path = list(self.path) + try: + for i in root.path: + path.remove(i) + except ValueError: + return Pather(path=self.path) + + return Pather(path=path) + + def startswith(self, path): + """ Returns True if path is the root of self Parameters: - seed (Union(bytes,str)): qb64b or qb64 serialization of private - signing key seed - """ - signer = Signer(qb64b=seed) - verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - return (pubkey == self.raw) + path (Pather): the path to check against self - def encrypt(self, ser=None, matter=None): - """ Returns: - Cipher instance of cipher text encryption of plain text serialization - provided by either ser or Matter instance when provided. + bool: True if path is the root of self - Parameters: - ser (Union[bytes,str]): qb64b or qb64 serialization of plain text - matter (Matter): plain text as Matter instance of seed or salt to - be encrypted """ - if not (ser or matter): - raise EmptyMaterialError("Neither ser or plain are provided.") - if ser: - matter = Matter(qb64b=ser) + return self.bext.startswith(path.bext) - if matter.code == MtrDex.Salt_128: # future other salt codes - code = MtrDex.X25519_Cipher_Salt - elif matter.code == MtrDex.Ed25519_Seed: # future other seed codes - code = MtrDex.X25519_Cipher_Seed - else: - raise ValueError("Unsupported plain text code = {}.".format(matter.code)) + def resolve(self, sad): + """ Recurses thru value following ptr - # encrypting fully qualified qb64 version of plain text ensures its - # derivation code round trips through eventual decryption - return (self._encrypt(ser=matter.qb64b, pubkey=self.raw, code=code)) + Parameters: + sad(dict or list): the next component - @staticmethod - def _x25519(ser, pubkey, code): + Returns: + Value at the end of the path """ - Returns cipher text as Cipher instance + return self._resolve(sad, self.path) + + + def tail(self, serder): + """ Recurses thru value following .path and returns terminal value + + Finds the value at this path and applies the version string rules of the serder + to serialize the value at ptr. + Parameters: - ser (Union[bytes, str]): qb64b or qb64 serialization of seed or salt - to be encrypted. - pubkey (bytes): raw binary serialization of encryption public key - code (str): derivation code of serialized plain text seed or salt + serder(Serder): the versioned dict to in which to resolve .path + + Returns: + bytes: Value at the end of the path """ - raw = pysodium.crypto_box_seal(ser, pubkey) - return Cipher(raw=raw, code=code) + val = self.resolve(sad=serder.sad) + if isinstance(val, str): + saider = Saider(qb64=val) + return saider.qb64b + elif isinstance(val, dict): + return dumps(val, serder.kind) + elif isinstance(val, list): + return dumps(val, serder.kind) + else: + raise ValueError("Bad tail value at {} of {}" + .format(self.path, serder.ked)) -class Decrypter(Matter): - """ - Decrypter is Matter subclass with method to decrypt the plain text from a - ciper text of a fully qualified (qb64) private key/seed where private - key/seed is the plain text. Decrypter uses assymetric (public, private) key - decryption of the cipher text using its .raw as the decrypting (private) key - and its .code to indicate the cipher suite for the decryption operation. - - For example .code == MtrDex.X25519 indicates that X25519 sealed box - decyrption is used. The decryption key may be derived from an Ed25519 - signing private key that is associated with a nontransferable or basic derivation - self certifying identifier. This allows use of the self certifying identifier - to track or manage the encryption/decryption key pair. And could be used to - provide additional authentication operations for using the - encryption/decryption key pair. Support for this is provided at init time - with the sigkey parameter which allows deriving the decryption private key - from the fully qualified sigkey (signing key). + @staticmethod + def _bextify(path): + """ Returns Base64 text delimited equivalent of path components - See Matter for inherited attributes and properties: + Suitable for variable sized matter - Attributes: - Properties: + Parameters: + path (list): array of path field components + Returns: + str: textual representation of SAD path - Methods: - decrypt: create cipher text + """ + vath = [] # valid path components + for p in path: + if hasattr(p, "decode"): + p = p.decode("utf-8") - """ + elif isinstance(p, int): + p = str(p) - def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): - """ - Assign decrypting cipher suite function to ._decrypt + if not Reb64.match(p.encode("utf-8")): + raise ValueError(f"Non Base64 path component = {p}.") - Parameters: See Matter for inheirted parameters - raw (bytes): private decryption key derived from seed (private signing key) - qb64b (bytes): fully qualified private decryption key - qb64 (str): fully qualified private decryption key - code (str): derivation code for private decryption key - seed (Union[bytes, str]): qb64b or qb64 of signing key seed used to - derive raw which is private decryption key - """ - try: - super(Decrypter, self).__init__(code=code, **kwa) - except EmptyMaterialError as ex: - if seed: - signer = Signer(qb64b=seed) - if signer.code not in (MtrDex.Ed25519_Seed,): - raise ValueError("Unsupported signing seed derivation code = {}." - "".format(signer.code)) - # verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) - sigkey = signer.raw + signer.verfer.raw # sigkey is raw seed + raw verkey - raw = pysodium.crypto_sign_sk_to_box_sk(sigkey) # raw private encrypt key - super(Decrypter, self).__init__(raw=raw, code=code, **kwa) - else: - raise + vath.append(p) - if self.code == MtrDex.X25519_Private: - self._decrypt = self._x25519 - else: - raise ValueError("Unsupported decrypter code = {}.".format(self.code)) + return ("-" + "-".join(vath)) - def decrypt(self, ser=None, cipher=None, transferable=False): - """ - Returns: - Salter or Signer instance derived from plain text decrypted from - encrypted cipher text material given by ser or cipher. Plain text - that is orignally encrypt should always be fully qualified (qb64b) - so that derivaton code of plain text is preserved through - encryption/decryption round trip. + def _resolve(self, val, ptr): + """ Recurses thru value following ptr Parameters: - ser (Union[bytes,str]): qb64b or qb64 serialization of cipher text - cipher (Cipher): optional Cipher instance when ser is None - transferable (bool): True means associated verfer of returned - signer is transferable. False means non-transferable + val(Optional(dict,list)): the next component + ptr(list): list of path components + + Returns: + Value at the end of the chain """ - if not (ser or cipher): - raise EmptyMaterialError("Neither ser or cipher are provided.") - if ser: # create cipher to ensure valid derivation code of material in ser - cipher = Cipher(qb64b=ser) + if len(ptr) == 0: + return val - return (self._decrypt(cipher=cipher, - prikey=self.raw, - transferable=transferable)) + idx = ptr.pop(0) - @staticmethod - def _x25519(cipher, prikey, transferable=False): - """ - Returns plain text as Salter or Signer instance depending on the cipher - code and the embedded encrypted plain text derivation code. + if isinstance(val, dict): + if idx.isdigit(): + i = int(idx) + + keys = list(val) + if i >= len(keys): + raise KeyError(f"invalid dict pointer index {i} for keys {keys}") + + cur = val[list(val)[i]] + elif idx == "": + return val + else: + cur = val[idx] + + elif isinstance(val, list): + i = int(idx) + if i >= len(val): + raise KeyError(f"invalid array pointer index {i} for array {val}") + + cur = val[i] - Parameters: - cipher (Cipher): instance of encrypted seed or salt - prikey (bytes): raw binary decryption private key derived from - signing seed or sigkey - transferable (bool): True means associated verfer of returned - signer is transferable. False means non-transferable - """ - pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) - plain = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) # qb64b - # ensure raw plain text is qb64b or qb64 so its derivation code is round tripped - if cipher.code == MtrDex.X25519_Cipher_Salt: - return Salter(qb64b=plain) - elif cipher.code == MtrDex.X25519_Cipher_Seed: - return Signer(qb64b=plain, transferable=transferable) else: - raise ValueError("Unsupported cipher text code = {}.".format(cipher.code)) + raise KeyError("invalid traversal type") + + return self._resolve(cur, ptr) -class Diger(Matter): +class Labeler(Matter): """ - Diger is Matter subclass with method to verify digest of serialization + Labeler is subclass of Matter for CESR native field map labels and/or generic + textual field values. Labeler auto sizes the instance code to minimize + the total encoded size of associated field label or textual field value. - See Matter for inherited attributes and properties: + Attributes: - Methods: - verify: verifies digest given ser - compare: compares provide digest given ser to this digest of ser. - enables digest agility of different digest algos to compare. + Inherited Properties: + (See Matter) - """ + Properties: + label (str): base value without encoding - def __init__(self, raw=None, ser=None, code=MtrDex.Blake3_256, **kwa): - """ - Assign digest verification function to ._verify + Inherited Hidden: + (See Matter) + + Hidden: + + Methods: + + """ - See Matter for inherited parameters + def __init__(self, label='', raw=None, code=None, soft=None, **kwa): + """ Inherited Parameters: - raw is bytes of unqualified crypto material usable for crypto operations - qb64b is bytes of fully qualified crypto material - qb64 is str or bytes of fully qualified crypto material - qb2 is bytes of fully qualified crypto material - code is str of derivation code - index is int of count of attached receipts for CryCntDex codes + (see Matter) Parameters: - ser is bytes serialization from which raw is computed if not raw + label (str | bytes): base value before encoding """ - # Should implement all digests in DigCodex instance DigDex - try: - super(Diger, self).__init__(raw=raw, code=code, **kwa) - except EmptyMaterialError as ex: - if not ser: - raise ex - if code == MtrDex.Blake3_256: - dig = blake3.blake3(ser).digest() - elif code == MtrDex.Blake2b_256: - dig = hashlib.blake2b(ser, digest_size=32).digest() - elif code == MtrDex.Blake2s_256: - dig = hashlib.blake2s(ser, digest_size=32).digest() - elif code == MtrDex.SHA3_256: - dig = hashlib.sha3_256(ser).digest() - elif code == MtrDex.SHA2_256: - dig = hashlib.sha256(ser).digest() + if label: + if hasattr(label, "encode"): # make label bytes + label = label.encode("utf-8") + + if Reb64.match(label): # candidate for Base64 compact encoding + try: + code = Tagger._codify(tag=label) + soft = label + + except InvalidSoftError as ex: # too big + if label[0] != ord(b'A'): # use Bexter code + code = LabelDex.StrB64_L0 + raw = Bexter._rawify(label) + + else: # use Texter code since ambiguity if starts with 'A' + code = LabelDex.Bytes_L0 + raw = label + else: - raise InvalidValueError("Unsupported code={code} for diger.") - - super(Diger, self).__init__(raw=dig, code=code, **kwa) - - if self.code == MtrDex.Blake3_256: - self._verify = self._blake3_256 - elif self.code == MtrDex.Blake2b_256: - self._verify = self._blake2b_256 - elif self.code == MtrDex.Blake2s_256: - self._verify = self._blake2s_256 - elif self.code == MtrDex.SHA3_256: - self._verify = self._sha3_256 - elif self.code == MtrDex.SHA2_256: - self._verify = self._sha2_256 - else: - raise InvalidValueError("Unsupported code={self.code} for diger.") + if len(label) == 1: + code = LabelDex.Label1 - def verify(self, ser): - """ - Returns True if raw digest of ser bytes (serialization) matches .raw - using .raw as reference digest for ._verify digest algorithm determined - by .code + elif len(label) == 2: + code = LabelDex.Label2 - Parameters: - ser (bytes): serialization to be digested and compared to .ser - """ - return (self._verify(ser=ser, raw=self.raw)) + else: + code = LabelDex.Bytes_L0 - def compare(self, ser, dig=None, diger=None): - """ - Returns True if dig and .qb64 or .qb64b match or - if both .raw and dig are valid digests of ser - Otherwise returns False + raw = label - Parameters: - ser is bytes serialization - dig is qb64b or qb64 digest of ser to compare with self - diger is Diger instance of digest of ser to compare with self + super(Labeler, self).__init__(raw=raw, code=code, soft=soft, **kwa) - if both supplied dig takes precedence + if self.code not in LabelDex: + raise InvalidCodeError(f"Invalid code={self.code} for Labeler.") - If both match then as optimization returns True and does not verify either - as digest of ser - Else If both have same code but do not match then as optimization returns False - and does not verify if either is digest of ser - Else recalcs both digests using each one's code to verify they - they are both digests of ser regardless of matching codes. + + @property + def label(self): + """Extracts and returns label from .code and .soft or .code and .raw + + Returns: + label (str): base value without encoding """ - if dig is not None: - if hasattr(dig, "encode"): - dig = dig.encode('utf-8') # makes bytes + if self.code in TagDex: # tag + return self.soft # soft part of code - if dig == self.qb64b: # matching - return True + if self.code in BexDex: # bext + return Bexter._derawify(raw=self.raw, code=self.code) # derawify - diger = Diger(qb64b=dig) # extract code + return self.raw.decode() # everything else is just raw as str - elif diger is not None: - if diger.qb64b == self.qb64b: - return True - else: - raise ValueError("Both dig and diger may not be None.") - if diger.code == self.code: # digest not match but same code - return False +class Verfer(Matter): + """ + Verfer is Matter subclass with method to verify signature of serialization + using the .raw as verifier key and .code for signature cipher suite. - if diger.verify(ser=ser) and self.verify(ser=ser): # both verify on ser - return True + See Matter for inherited attributes and properties: - return (False) + Attributes: - @staticmethod - def _blake3_256(ser, raw): + Properties: + + Methods: + verify: verifies signature + + """ + + def __init__(self, **kwa): """ - Returns True if verified False otherwise - Verifiy blake3_256 digest of ser matches raw + Assign verification cipher suite function to ._verify - Parameters: - ser is bytes serialization - dig is bytes reference digest """ - return (blake3.blake3(ser).digest() == raw) + super(Verfer, self).__init__(**kwa) - @staticmethod - def _blake2b_256(ser, raw): + if self.code in [MtrDex.Ed25519N, MtrDex.Ed25519]: + self._verify = self._ed25519 + elif self.code in [MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256r1]: + self._verify = self._secp256r1 + elif self.code in [MtrDex.ECDSA_256k1N, MtrDex.ECDSA_256k1]: + self._verify = self._secp256k1 + else: + raise ValueError("Unsupported code = {} for verifier.".format(self.code)) + + def verify(self, sig, ser): """ - Returns True if verified False otherwise - Verifiy blake2b_256 digest of ser matches raw + Returns True if bytes signature sig verifies on bytes serialization ser + using .raw as verifier public key for ._verify cipher suite determined + by .code Parameters: + sig is bytes signature ser is bytes serialization - dig is bytes reference digest """ - return (hashlib.blake2b(ser, digest_size=32).digest() == raw) + return (self._verify(sig=sig, ser=ser, key=self.raw)) @staticmethod - def _blake2s_256(ser, raw): + def _ed25519(sig, ser, key): """ Returns True if verified False otherwise - Verifiy blake2s_256 digest of ser matches raw + Verify Ed25519 sig on ser using key Parameters: + sig is bytes signature ser is bytes serialization - dig is bytes reference digest + key is bytes public key """ - return (hashlib.blake2s(ser, digest_size=32).digest() == raw) + try: # verify returns None if valid else raises ValueError + pysodium.crypto_sign_verify_detached(sig, ser, key) + except Exception as ex: + return False + + return True @staticmethod - def _sha3_256(ser, raw): + def _secp256r1(sig, ser, key): """ Returns True if verified False otherwise - Verifiy blake2s_256 digest of ser matches raw + Verify secp256r1 sig on ser using key Parameters: + sig is bytes signature ser is bytes serialization - dig is bytes reference digest + key is bytes public key """ - return (hashlib.sha3_256(ser).digest() == raw) + verkey = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256R1(), key) + r = int.from_bytes(sig[:32], "big") + s = int.from_bytes(sig[32:], "big") + der = utils.encode_dss_signature(r, s) + try: + verkey.verify(der, ser, ec.ECDSA(hashes.SHA256())) + return True + except exceptions.InvalidSignature: + return False @staticmethod - def _sha2_256(ser, raw): + def _secp256k1(sig, ser, key): """ Returns True if verified False otherwise - Verifiy blake2s_256 digest of ser matches raw + Verify secp256k1 sig on ser using key Parameters: + sig is bytes signature ser is bytes serialization - dig is bytes reference digest + key is bytes public key """ - return (hashlib.sha256(ser).digest() == raw) - + verkey = ec.EllipticCurvePublicKey.from_encoded_point(ec.SECP256K1(), key) + r = int.from_bytes(sig[:32], "big") + s = int.from_bytes(sig[32:], "big") + der = utils.encode_dss_signature(r, s) + try: + verkey.verify(der, ser, ec.ECDSA(hashes.SHA256())) + return True + except exceptions.InvalidSignature: + return False -@dataclass(frozen=True) -class PreCodex: - """ - PreCodex is codex all identifier prefix derivation codes. - This is needed to verify valid inception events. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. +class Cigar(Matter): """ - Ed25519N: str = 'B' # Ed25519 verification key non-transferable, basic derivation. - Ed25519: str = 'D' # Ed25519 verification key basic derivation - Blake3_256: str = 'E' # Blake3 256 bit digest self-addressing derivation. - Blake2b_256: str = 'F' # Blake2b 256 bit digest self-addressing derivation. - Blake2s_256: str = 'G' # Blake2s 256 bit digest self-addressing derivation. - SHA3_256: str = 'H' # SHA3 256 bit digest self-addressing derivation. - SHA2_256: str = 'I' # SHA2 256 bit digest self-addressing derivation. - Blake3_512: str = '0D' # Blake3 512 bit digest self-addressing derivation. - Blake2b_512: str = '0E' # Blake2b 512 bit digest self-addressing derivation. - SHA3_512: str = '0F' # SHA3 512 bit digest self-addressing derivation. - SHA2_512: str = '0G' # SHA2 512 bit digest self-addressing derivation. - ECDSA_256k1N: str = '1AAA' # ECDSA secp256k1 verification key non-transferable, basic derivation. - ECDSA_256k1: str = '1AAB' # ECDSA public verification or encryption key, basic derivation - ECDSA_256r1N: str = "1AAI" # ECDSA secp256r1 verification key non-transferable, basic derivation. - ECDSA_256r1: str = "1AAJ" # ECDSA secp256r1 verification or encryption key, basic derivation - - def __iter__(self): - return iter(astuple(self)) - - -PreDex = PreCodex() # Make instance - + Cigar is Matter subclass holding a nonindexed signature with verfer property. + From Matter .raw is signature and .code is signature cipher suite + Adds .verfer property to hold Verfer instance of associated verifier public key + Verfer's .raw as verifier key and .code is verifier cipher suite. -class Prefixer(Matter): - """ - Prefixer is Matter subclass for autonomic identifier prefix using - derivation as determined by code from ked + See Matter for inherited attributes and properties: Attributes: - Inherited Properties: (see Matter) - .pad is int number of pad chars given raw - .code is str derivation code to indicate cypher suite + Properties: (Inherited) + .code is str derivation code to indicate cypher suite + .size is size (int): number of quadlets when variable sized material besides + full derivation code else None .raw is bytes crypto material only without code - .index is int count of attached crypto material by context (receipts) .qb64 is str in Base64 fully qualified with derivation code + crypto mat .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat .qb2 is bytes in binary with derivation code + crypto material .transferable is Boolean, True when transferable derivation code False otherwise + .digestive is Boolean, True when digest derivation code False otherwise Properties: + .verfer is verfer of public key used to verify signature + + Hidden: + ._code is str value for .code property + ._size is int value for .size property + ._raw is bytes value for .raw property + ._infil is method to compute fully qualified Base64 from .raw and .code + ._exfil is method to extract .code and .raw from fully qualified Base64 Methods: - verify(): Verifies derivation of aid prefix from a ked Hidden: ._pad is method to compute .pad property @@ -3359,278 +3214,196 @@ class Prefixer(Matter): ._index is int value for .index property ._infil is method to compute fully qualified Base64 from .raw and .code ._exfil is method to extract .code and .raw from fully qualified Base64 + """ - Dummy = "#" # dummy spaceholder char for pre. Must not be a valid Base64 char - def __init__(self, raw=None, code=None, ked=None, allows=None, **kwa): + def __init__(self, verfer=None, **kwa): """ - assign ._derive to derive aid prefix from ked - assign ._verify to verify derivation of aid prefix from ked - - Default code is None to force EmptyMaterialError when only raw provided but - not code. - - Inherited Parameters: - raw is bytes of unqualified crypto material usable for crypto operations - qb64b is bytes of fully qualified crypto material - qb64 is str or bytes of fully qualified crypto material - qb2 is bytes of fully qualified crypto material - code is str of derivation code - index is int of count of attached receipts for CryCntDex codes - - Parameters: - allows (list): allowed codes for prefix. When None then all supported - codes are allowed. This enables a particular use case to restrict - the codes allowed to a subset of all supported. + Assign verfer to ._verfer attribute """ - try: - super(Prefixer, self).__init__(raw=raw, code=code, **kwa) - except EmptyMaterialError as ex: - if not ked or (not code and "i" not in ked): - raise ex - - if not code: # get code from pre in ked - super(Prefixer, self).__init__(qb64=ked["i"], code=code, **kwa) - code = self.code - - if allows is not None and code not in allows: - raise ValueError("Unallowed code={} for prefixer.".format(code)) - - if code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: - self._derive = self._derive_non_transferable - elif code in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: - self._derive = self._derive_transferable - elif code == MtrDex.Blake3_256: - self._derive = self._derive_blake3_256 - else: - raise ValueError("Unsupported code = {} for prefixer.".format(code)) - - # use ked and ._derive from code to derive aid prefix and code - raw, code = self.derive(ked=ked) - super(Prefixer, self).__init__(raw=raw, code=code, **kwa) - - if self.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: - self._verify = self._verify_non_transferable - elif self.code in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: - self._verify = self._verify_transferable - elif self.code == MtrDex.Blake3_256: - self._verify = self._verify_blake3_256 - else: - raise ValueError("Unsupported code = {} for prefixer.".format(self.code)) + super(Cigar, self).__init__(**kwa) + self._verfer = verfer - def derive(self, ked): + @property + def verfer(self): """ - Returns tuple (raw, code) of aid prefix as derived from key event dict ked. - uses a derivation code specific _derive method - - Parameters: - ked is inception key event dict - seed is only used for sig derivation it is the secret key/secret - + Property verfer: + Returns Verfer instance + Assumes ._verfer is correctly assigned """ - ilk = ked["t"] - if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): - raise ValueError("Nonincepting ilk={} for prefix derivation.".format(ilk)) + return self._verfer - labels = getattr(Labels, ilk) - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) + @verfer.setter + def verfer(self, verfer): + """ verfer property setter """ + self._verfer = verfer - return (self._derive(ked=ked)) - def verify(self, ked, prefixed=False): - """ - Returns True if derivation from ked for .code matches .qb64 and - If prefixed also verifies ked["i"] matches .qb64 - False otherwise - Parameters: - ked is inception key event dict - """ - ilk = ked["t"] - if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): - raise ValueError("Nonincepting ilk={} for prefix derivation.".format(ilk)) +class Diger(Matter): + """ + Diger is Matter subclass with method to verify digest of serialization - labels = getattr(Labels, ilk) - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) - return (self._verify(ked=ked, pre=self.qb64, prefixed=prefixed)) + See Matter for inherited attributes and properties: - def _derive_non_transferable(self, ked): - """ - Returns tuple (raw, code) of basic nontransferable Ed25519 prefix (qb64) - as derived from inception key event dict ked keys[0] - """ - ked = dict(ked) # make copy so don't clobber original ked - try: - keys = ked["k"] - if len(keys) != 1: - raise DerivationError("Basic derivation needs at most 1 key " - " got {} keys instead".format(len(keys))) - verfer = Verfer(qb64=keys[0]) - except Exception as ex: - raise DerivationError("Error extracting public key =" - " = {}".format(ex)) - if verfer.code not in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N]: - raise DerivationError("Mismatch derivation code = {}." - "".format(verfer.code)) + Methods: + verify: verifies digest given ser + compare: compares provide digest given ser to this digest of ser. + enables digest agility of different digest algos to compare. - try: - if verfer.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N] and ked["n"]: - raise DerivationError("Non-empty nxt = {} for non-transferable" - " code = {}".format(ked["n"], - verfer.code)) - if verfer.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N] and "b" in ked and ked["b"]: - raise DerivationError("Non-empty b = {} for non-transferable" - " code = {}".format(ked["b"], - verfer.code)) + """ - if verfer.code in [MtrDex.Ed25519N, MtrDex.ECDSA_256r1N, MtrDex.ECDSA_256k1N] and "a" in ked and ked["a"]: - raise DerivationError("Non-empty a = {} for non-transferable" - " code = {}".format(ked["a"], - verfer.code)) + # Maps digest codes to Digestages of algorithms for computing digest. + # Should be based on the same set of codes as in DigestCodex + # so Matter.digestive property works. + # Use unit tests to ensure codex elements sets match - except Exception as ex: - raise DerivationError("Error checking nxt = {}".format(ex)) + Digests = { + DigDex.Blake3_256: Digestage(klas=blake3.blake3, size=None, length=None), + DigDex.Blake2b_256: Digestage(klas=hashlib.blake2b, size=32, length=None), + DigDex.Blake2s_256: Digestage(klas=hashlib.blake2s, size=None, length=None), + DigDex.SHA3_256: Digestage(klas=hashlib.sha3_256, size=None, length=None), + DigDex.SHA2_256: Digestage(klas=hashlib.sha256, size=None, length=None), + DigDex.Blake3_512: Digestage(klas=blake3.blake3, size=None, length=64), + DigDex.Blake2b_512: Digestage(klas=hashlib.blake2b, size=None, length=None), + DigDex.SHA3_512: Digestage(klas=hashlib.sha3_512, size=None, length=None), + DigDex.SHA2_512: Digestage(klas=hashlib.sha512, size=None, length=None), + } - return (verfer.raw, verfer.code) + def __init__(self, raw=None, ser=None, code=DigDex.Blake3_256, **kwa): + """Initialize attributes - def _verify_non_transferable(self, ked, pre, prefixed=False): - """ - Returns True if verified False otherwise - Verify derivation of fully qualified Base64 pre from inception iked dict + Inherited Parameters: + See Matter Parameters: - ked is inception key event dict - pre is Base64 fully qualified prefix default to .qb64 + ser (bytes): serialization from which raw is computed if not raw + """ + try: - keys = ked["k"] - if len(keys) != 1: - return False - - if keys[0] != pre: - return False + super(Diger, self).__init__(raw=raw, code=code, **kwa) + except EmptyMaterialError as ex: + if not ser: + raise ex - if prefixed and ked["i"] != pre: - return False + raw = self._digest(ser, code=code) - if ked["n"]: # must be empty - return False + super(Diger, self).__init__(raw=raw, code=code, **kwa) - except Exception as ex: - return False + if self.code not in DigDex: + raise InvalidCodeError(f"Unsupported Digest {code=}.") - return True + @classmethod + def _digest(cls, ser, code=DigDex.Blake3_256): + """Returns raw digest of ser using digest algorithm given by code - def _derive_transferable(self, ked): + Parameters: + ser (bytes): serialization from which raw digest is computed + code (str): derivation code used to lookup digest algorithm """ - Returns tuple (raw, code) of basic Ed25519 prefix (qb64) - as derived from inception key event dict ked keys[0] + if code not in cls.Digests: + raise InvalidCodeError(f"Unsupported Digest {code=}.") + + klas, size, length = cls.Digests[code] # digest algo size & length + ikwa = dict(digest_size=size) if size else dict() # opt digest size + dkwa = dict(length=length) if length else dict() # opt digest length + raw = klas(ser, **ikwa).digest(**dkwa) + return (raw) + + + def verify(self, ser): """ - ked = dict(ked) # make copy so don't clobber original ked - try: - keys = ked["k"] - if len(keys) != 1: - raise DerivationError("Basic derivation needs at most 1 key " - " got {} keys instead".format(len(keys))) - verfer = Verfer(qb64=keys[0]) - except Exception as ex: - raise DerivationError("Error extracting public key =" - " = {}".format(ex)) + Returns True if raw digest of ser bytes (serialization) matches .raw + using .raw as reference digest for digest algorithm determined + by .code + + Parameters: + ser (bytes): serialization to be digested and compared to .raw - if verfer.code not in [MtrDex.Ed25519, MtrDex.ECDSA_256r1, MtrDex.ECDSA_256k1]: - raise DerivationError("Mismatch derivation code = {}" - "".format(verfer.code)) + """ + return (self._digest(ser=ser, code=self.code) == self.raw) - return (verfer.raw, verfer.code) - def _verify_transferable(self, ked, pre, prefixed=False): + def compare(self, ser, dig=None, diger=None): """ - Returns True if verified False otherwise - Verify derivation of fully qualified Base64 prefix from - inception key event dict (ked) + Returns True if dig and .qb64 or .qb64b match or + if both .raw and dig are valid digests of ser + Otherwise returns False Parameters: - ked is inception key event dict - pre is Base64 fully qualified prefix default to .qb64 + ser is bytes serialization + dig is qb64b or qb64 digest of ser to compare with self + diger is Diger instance of digest of ser to compare with self + + if both supplied dig takes precedence + + + If both match then as optimization returns True and does not verify either + as digest of ser + Else If both have same code but do not match then as optimization returns False + and does not verify if either is digest of ser + Else recalcs both digests using each one's code to verify they + they are both digests of ser regardless of matching codes. """ - try: - keys = ked["k"] - if len(keys) != 1: - return False + if dig is not None: + if hasattr(dig, "encode"): + dig = dig.encode('utf-8') # makes bytes - if keys[0] != pre: - return False + if dig == self.qb64b: # matching + return True - if prefixed and ked["i"] != pre: - return False + diger = Diger(qb64b=dig) # extract code - except Exception as ex: + elif diger is not None: + if diger.qb64b == self.qb64b: + return True + + else: + raise ValueError("Both dig and diger may not be None.") + + if diger.code == self.code: # digest not match but same code return False - return True + if diger.verify(ser=ser) and self.verify(ser=ser): # both verify on ser + return True + return (False) - def _derive_blake3_256(self, ked): - """ - Returns tuple (raw, code) of pre (qb64) as blake3 digest - as derived from inception key event dict ked - """ - ked = dict(ked) # make copy so don't clobber original ked - ilk = ked["t"] - if ilk not in (Ilks.icp, Ilks.dip, Ilks.vcp, Ilks.iss): - raise DerivationError("Invalid ilk = {} to derive pre.".format(ilk)) - # put in dummy pre to get size correct - ked["i"] = self.Dummy * Matter.Sizes[MtrDex.Blake3_256].fs - ked["d"] = ked["i"] # must be same dummy - #raw, proto, kind, ked, version = sizeify(ked=ked) - raw, _, _, _, _ = sizeify(ked=ked) - dig = blake3.blake3(raw).digest() # digest with dummy 'i' and 'd' - return (dig, MtrDex.Blake3_256) # dig is derived correct new 'i' and 'd' +class Prefixer(Matter): + """ + Prefixer is Matter subclass for autonomic identifier AID prefix - def _verify_blake3_256(self, ked, pre, prefixed=False): - """ - Returns True if verified False otherwise - Verify derivation of fully qualified Base64 prefix from - inception key event dict (ked) + Attributes: - Parameters: - ked is inception key event dict - pre is Base64 fully qualified default to .qb64 - """ - try: - raw, code = self._derive_blake3_256(ked=ked) # replace with dummy 'i' - crymat = Matter(raw=raw, code=MtrDex.Blake3_256) - if crymat.qb64 != pre: # derived raw with dummy 'i' must match pre - return False + Inherited Properties: (see Matter) - if prefixed and ked["i"] != pre: # incoming 'i' must match pre - return False + Properties: - if ked["i"] != ked["d"]: # when digestive then SAID must match pre - return False + Methods: - except Exception as ex: - return False + Hidden: - return True + """ + + def __init__(self, **kwa): + """Checks for .code in PreDex so valid prefixive code + Inherited Parameters: + See Matter + + """ + super(Prefixer, self).__init__(**kwa) + if self.code not in PreDex: + raise InvalidCodeError(f"Invalid prefixer code = {self.code}.") -# digest algorithm klas, digest size (not default), digest length -# size and length are needed for some digest types as function parameters -Digestage = namedtuple("Digestage", "klas size length") class Saider(Matter): @@ -3662,19 +3435,6 @@ class Saider(Matter): """ Dummy = "#" # dummy spaceholder char for said. Must not be a valid Base64 char - # should be same set of codes as in coring.DigestCodex coring.DigDex so - # .digestive property works. Unit test ensures code sets match - Digests = { - MtrDex.Blake3_256: Digestage(klas=blake3.blake3, size=None, length=None), - MtrDex.Blake2b_256: Digestage(klas=hashlib.blake2b, size=32, length=None), - MtrDex.Blake2s_256: Digestage(klas=hashlib.blake2s, size=None, length=None), - MtrDex.SHA3_256: Digestage(klas=hashlib.sha3_256, size=None, length=None), - MtrDex.SHA2_256: Digestage(klas=hashlib.sha256, size=None, length=None), - MtrDex.Blake3_512: Digestage(klas=blake3.blake3, size=None, length=64), - MtrDex.Blake2b_512: Digestage(klas=hashlib.blake2b, size=None, length=None), - MtrDex.SHA3_512: Digestage(klas=hashlib.sha3_512, size=None, length=None), - MtrDex.SHA2_512: Digestage(klas=hashlib.sha512, size=None, length=None), - } def __init__(self, raw=None, *, code=None, sad=None, kind=None, label=Saids.d, ignore=None, **kwa): @@ -3733,1640 +3493,158 @@ def _serialize(clas, sad: dict, kind: str = None): Parameters: sad (dict): serializable dict - kind (str): serialization algorithm of sad, one of Serials - used to override that given by 'v' field if any in sad - otherwise default is Serials.json - - """ - knd = Serials.json - if 'v' in sad: # versioned sad - _, _, knd, _ = deversify(sad['v']) - - if not kind: # match logic of Serder for kind - kind = knd - - return dumps(sad, kind=kind) - - @classmethod - def saidify(clas, - sad: dict, - *, - code: str = MtrDex.Blake3_256, - kind: str = None, - label: str = Saids.d, - ignore: list = None, **kwa): - """ - Derives said from sad and injects it into copy of sad and said and - injected sad - - Returns: - result (tuple): of the form (saider, sad) where saider is Saider - instance generated from sad using code and sad is copy of - parameter sad but with its label id field filled - in with generated said from saider - - Parameters: - sad (dict): serializable dict - code (str): digest type code from DigDex - kind (str): serialization algorithm of sad, one of Serials - used to override that given by 'v' field if any in sad - otherwise default is Serials.json - label (str): Saidage value as said field label in which to inject said - ignore (list): fields to ignore when generating SAID - - """ - if label not in sad: - raise KeyError("Missing id field labeled={} in sad.".format(label)) - raw, sad = clas._derive(sad=sad, code=code, kind=kind, label=label, ignore=ignore) - saider = clas(raw=raw, code=code, kind=kind, label=label, ignore=ignore, **kwa) - sad[label] = saider.qb64 - return saider, sad - - - @classmethod - def _derive(clas, sad: dict, *, - code: str = MtrDex.Blake3_256, - kind: str = None, - label: str = Saids.d, - ignore: list = None): - """ - Derives raw said from sad with .Dummy filled sad[label] - - Returns: - raw (bytes): raw said from sad with dummy filled label id field - - Parameters: - sad (dict): self addressed data to be injected with dummy and serialized - code (str): digest type code from DigDex - kind (str): serialization algorithm of sad, one of Serials - used to override that given by 'v' field if any in sad - otherwise default is Serials.json - label (str): Saidage value as said field label in which to inject dummy - ignore (list): fields to ignore when generating SAID - - """ - if code not in DigDex or code not in clas.Digests: - raise ValueError("Unsupported digest code = {}.".format(code)) - - sad = dict(sad) # make shallow copy so don't clobber original sad - # fill id field denoted by label with dummy chars to get size correct - sad[label] = clas.Dummy * Matter.Sizes[code].fs - if 'v' in sad: # if versioned then need to set size in version string - raw, proto, kind, sad, version = sizeify(ked=sad, kind=kind) - - ser = dict(sad) - if ignore: - for f in ignore: - del ser[f] - - # string now has - # correct size - klas, size, length = clas.Digests[code] - # sad as 'v' verision string then use its kind otherwise passed in kind - cpa = [clas._serialize(ser, kind=kind)] # raw pos arg class - ckwa = dict() # class keyword args - if size: - ckwa.update(digest_size=size) # optional digest_size - dkwa = dict() # digest keyword args - if length: - dkwa.update(length=length) - return klas(*cpa, **ckwa).digest(**dkwa), sad # raw digest and sad - - - def derive(self, sad, code=None, **kwa): - """ - Returns: - result (tuple): (raw, sad) raw said as derived from serialized dict - and modified sad during derivation. - - Parameters: - sad (dict): self addressed data to be serialized - code (str): digest type code from DigDex. - kind (str): serialization algorithm of sad, one of Serials - used to override that given by 'v' field if any in sad - otherwise default is Serials.json - label (str): Saidage value of said field labelin which to inject dummy - """ - code = code if code is not None else self.code - return self._derive(sad=sad, code=code, **kwa) - - - def verify(self, sad, *, prefixed=False, versioned=True, code=None, - kind=None, label=Saids.d, ignore=None, **kwa): - """ - Returns: - result (bool): True means derivation from sad with dummy label - field value replacement for ._code matches .qb64. False otherwise - If prefixed is True then also validates that label field of - provided sad also matches .qb64. False otherwise - If versioned is True and provided sad includes version field 'v' - then also validates that version field 'v' of provided - sad matches the version field of modified sad that results from - the derivation process. The size chars in the version field - are set to the size of the sad during derivation. False otherwise. - - Parameters: - sad (dict): self addressed data to be serialized - prefixed (bool): True means also verify if labeled field in - sad matches own .qb64 - versioned (bool): - code (str): digest type code from DigDex. - kind (str): serialization algorithm of sad, one of Serials - used to override that given by 'v' field if any in sad - otherwise default is Serials.json - label (str): Saidage value of said field label in which to inject dummy - ignore (list): fields to ignore when generating SAID - """ - try: - # override ensure code is self.code - raw, dsad = self._derive(sad=sad, code=self.code, kind=kind, label=label, ignore=ignore) - saider = Saider(raw=raw, code=self.code, ignore=ignore, **kwa) - if self.qb64b != saider.qb64b: - return False # not match .qb64b - - if 'v' in sad and versioned: - if sad['v'] != dsad['v']: - return False # version fields not match - - if prefixed and sad[label] != self.qb64: # check label field - return False # label id field not match .qb64 - - except Exception as ex: - return False - - return True - - -@dataclass(frozen=True) -class IndexerCodex: - """ IndexerCodex is codex hard (stable) part of all indexer derivation codes. - - Codes indicate which list of keys, current and/or prior next, index is for: - - _Sig: Indices in code may appear in both current signing and - prior next key lists when event has both current and prior - next key lists. Two character code table has only one index - so must be the same for both lists. Other index if for - prior next. - The indices may be different in those code tables which - have two sets of indices. - - _Crt_Sig: Index in code for current signing key list only. - - _Big_: Big index values - - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - - Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. - Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. - ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. - ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. - ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. - Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. - Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. - Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. - Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. - ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. - ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. - Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. - Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. - TBD0: str = '0z' # Test of Var len label L=N*4 <= 4095 char quadlets includes code - TBD1: str = '1z' # Test of index sig lead 1 - TBD4: str = '4z' # Test of index sig lead 1 big - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" - -IdrDex = IndexerCodex() - - -@dataclass(frozen=True) -class IndexedSigCodex: - """IndexedSigCodex is codex all indexed signature derivation codes. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. - Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. - ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. - ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. - ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. - Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. - Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. - Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. - Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. - ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. - ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. - Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. - Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. - - def __iter__(self): - return iter(astuple(self)) - -IdxSigDex = IndexedSigCodex() # Make instance - - -@dataclass(frozen=True) -class IndexedCurrentSigCodex: - """IndexedCurrentSigCodex is codex indexed signature codes for current list. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. - Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. - Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. - ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. - ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. - Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. - - def __iter__(self): - return iter(astuple(self)) - -IdxCrtSigDex = IndexedCurrentSigCodex() # Make instance - - - -@dataclass(frozen=True) -class IndexedBothSigCodex: - """IndexedBothSigCodex is codex indexed signature codes for both lists. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. - ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. - ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. - Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. - Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both listsy. - ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. - ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. - Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. - - def __iter__(self): - return iter(astuple(self)) - -IdxBthSigDex = IndexedBothSigCodex() # Make instance - -# namedtuple for size entries in Incexer derivation code tables -# hs is the hard size int number of chars in hard (stable) part of code -# ss is the soft size int number of chars in soft (unstable) part of code -# os is the other size int number of chars in other index part of soft -# ms = ss - os main index size computed -# fs is the full size int number of chars in code plus appended material if any -# ls is the lead size int number of bytes to pre-pad pre-converted raw binary -Xizage = namedtuple("Xizage", "hs ss os fs ls") - -class Indexer: - """ Indexer is fully qualified cryptographic material primitive base class for - indexed primitives. In special cases some codes in the Index code table - may be of variable length (i.e. not indexed) when the full size table entry - is None. In that case the index is used instread as the length. - - Sub classes are derivation code and key event element context specific. - - Includes the following attributes and properties: - - Attributes: - - Properties: - code is str of stable (hard) part of derivation code - raw (bytes): unqualified crypto material usable for crypto operations - index (int): main index offset into list or length of material - ondex (int | None): other index offset into list or length of material - qb64b (bytes): fully qualified Base64 crypto material - qb64 (str | bytes): fully qualified Base64 crypto material - qb2 (bytes): fully qualified binary crypto material - - Hidden: - ._code (str): value for .code property - ._raw (bytes): value for .raw property - ._index (int): value for .index property - ._ondex (int): value for .ondex property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._binfil is method to compute fully qualified Base2 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 - ._bexfil is method to extract .code and .raw from fully qualified Base2 - - """ - Codex = IdrDex - # Hards table maps from bytes Base64 first code char to int of hard size, hs, - # (stable) of code. The soft size, ss, (unstable) is always > 0 for Indexer. - Hards = ({chr(c): 1 for c in range(65, 65 + 26)}) - Hards.update({chr(c): 1 for c in range(97, 97 + 26)}) - Hards.update([('0', 2), ('1', 2), ('2', 2), ('3', 2), ('4', 2)]) - # Sizes table maps hs chars of code to Xizage namedtuple of (hs, ss, os, fs, ls) - # where hs is hard size, ss is soft size, os is other index size, - # and fs is full size, ls is lead size. - # where ss includes os, so main index size ms = ss - os - # soft size, ss, should always be > 0 for Indexer - Sizes = { - 'A': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'B': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'C': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'D': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'E': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'F': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - '0A': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '0B': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '2A': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2B': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2C': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2D': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2E': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2F': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '3A': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '3B': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '0z': Xizage(hs=2, ss=2, os=0, fs=None, ls=0), - '1z': Xizage(hs=2, ss=2, os=1, fs=76, ls=1), - '4z': Xizage(hs=2, ss=6, os=3, fs=80, ls=1), - } - # Bards table maps to hard size, hs, of code from bytes holding sextets - # converted from first code char. Used for ._bexfil. - Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) - - def __init__(self, raw=None, code=IdrDex.Ed25519_Sig, index=0, ondex=None, - qb64b=None, qb64=None, qb2=None, strip=False): - """ - Validate as fully qualified - Parameters: - raw (bytes): unqualified crypto material usable for crypto operations - code is str of stable (hard) part of derivation code - index (int): main index offset into list or length of material - ondex (int | None): other index offset into list or length of material - qb64b (bytes): fully qualified Base64 crypto material - qb64 (str | bytes): fully qualified Base64 crypto material - qb2 (bytes): fully qualified binary crypto material - strip (bool): True means strip counter contents from input stream - bytearray after parsing qb64b or qb2. False means do not strip - - Needs either (raw and code and index) or qb64b or qb64 or qb2 - Otherwise raises EmptyMaterialError - When raw and code provided then validate that code is correct - for length of raw and assign .raw - Else when qb64b or qb64 or qb2 provided extract and assign - .raw, .code, .index, .ondex. - - """ - if raw is not None: # raw provided - if not code: - raise EmptyMaterialError("Improper initialization need either " - "(raw and code) or qb64b or qb64 or qb2.") - if not isinstance(raw, (bytes, bytearray)): - raise TypeError(f"Not a bytes or bytearray, raw={raw}.") - - if code not in self.Sizes: - raise UnexpectedCodeError(f"Unsupported code={code}.") - - hs, ss, os, fs, ls = self.Sizes[code] # get sizes for code - cs = hs + ss # both hard + soft code size - ms = ss - os - - if not isinstance(index, int) or index < 0 or index > (64 ** ms - 1): - raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") - - if isinstance(ondex, int) and os and not (ondex >= 0 and ondex <= (64 ** os - 1)): - raise InvalidVarIndexError(f"Invalid ondex={ondex} for code={code}.") - - if code in IdxCrtSigDex and ondex is not None: - raise InvalidVarIndexError(f"Non None ondex={ondex} for code={code}.") - - if code in IdxBthSigDex: - if ondex is None: # set default - ondex = index # when not provided make ondex match index - else: - if ondex != index and os == 0: # must match if os == 0 - raise InvalidVarIndexError(f"Non matching ondex={ondex}" - f" and index={index} for " - f"code={code}.") - - - if not fs: # compute fs from index - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise InvalidCodeSizeError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - - rawsize = (fs - cs) * 3 // 4 - - raw = raw[:rawsize] # copy rawsize from stream, may be less - if len(raw) != rawsize: # forbids shorter - raise RawMaterialError(f"Not enougth raw bytes for code={code}" - f"and index={index} ,expected {rawsize} " - f"got {len(raw)}.") - - self._code = code - self._index = index - self._ondex = ondex - self._raw = bytes(raw) # crypto ops require bytes not bytearray - - elif qb64b is not None: - self._exfil(qb64b) - if strip: # assumes bytearray - del qb64b[:len(self.qb64b)] # may be variable length fs - - elif qb64 is not None: - self._exfil(qb64) - - elif qb2 is not None: - self._bexfil(qb2) - if strip: # assumes bytearray - del qb2[:len(self.qb2)] # may be variable length fs - - else: - raise EmptyMaterialError("Improper initialization need either " - "(raw and code and index) or qb64b or " - "qb64 or qb2.") - - @classmethod - def _rawSize(cls, code): - """ - Returns expected raw size in bytes for a given code. Not applicable to - codes with fs = None - """ - hs, ss, os, fs, ls = cls.Sizes[code] # get sizes - return ((fs - (hs + ss)) * 3 // 4) - - @property - def code(self): - """ - Returns ._code - Makes .code read only - """ - return self._code - - @property - def raw(self): - """ - Returns ._raw - Makes .raw read only - """ - return self._raw - - @property - def index(self): - """ - Returns ._index - Makes .index read only - """ - return self._index - - @property - def ondex(self): - """ - Returns ._ondex - Makes .ondex read only - """ - return self._ondex - - @property - def qb64b(self): - """ - Property qb64b: - Returns Fully Qualified Base64 Version encoded as bytes - Assumes self.raw and self.code are correctly populated - """ - return self._infil() - - @property - def qb64(self): - """ - Property qb64: - Returns Fully Qualified Base64 Version - Assumes self.raw and self.code are correctly populated - """ - return self.qb64b.decode("utf-8") - - @property - def qb2(self): - """ - Property qb2: - Returns Fully Qualified Binary Version Bytes - """ - return self._binfil() - - def _infil(self): - """ - Returns fully qualified attached sig base64 bytes computed from - self.raw, self.code and self.index. - - cs = hs + ss - os = ss - ms (main index size) - when fs None then size computed & fs = size * 4 + cs - - """ - code = self.code # codex value chars hard code - index = self.index # main index value - ondex = self.ondex # other index value - raw = self.raw # bytes or bytearray - - ps = (3 - (len(raw) % 3)) % 3 # if lead then same pad size chars & lead size bytes - hs, ss, os, fs, ls = self.Sizes[code] - cs = hs + ss - ms = ss - os - - if not fs: # compute fs from index - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise InvalidCodeSizeError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - - if index < 0 or index > (64 ** ms - 1): - raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") - - if (isinstance(ondex, int) and os and - not (ondex >= 0 and ondex <= (64 ** os - 1))): - raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " - f"code={code}.") - - # both is hard code + converted index + converted ondex - both = (f"{code}{intToB64(index, l=ms)}" - f"{intToB64(ondex if ondex is not None else 0, l=os)}") - - # check valid pad size for whole code size, assumes ls is zero - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(both))) - - if (cs % 4) != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - - # prepend pad bytes, convert, then replace pad chars with full derivation - # code including index, - full = both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[ps - ls:] - - if len(full) != fs: # invalid size - raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") - - return full - - - def _binfil(self): - """ - Returns bytes of fully qualified base2 bytes, that is .qb2 - self.code and self.index converted to Base2 + self.raw left shifted - with pad bits equivalent of Base64 decode of .qb64 into .qb2 - """ - code = self.code # codex chars hard code - index = self.index # main index value - ondex = self.ondex # other index value - raw = self.raw # bytes or bytearray - - ps = (3 - (len(raw) % 3)) % 3 # same pad size chars & lead size bytes - hs, ss, os, fs, ls = self.Sizes[code] - cs = hs + ss - ms = ss - os - - if index < 0 or index > (64 ** ss - 1): - raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") - - if (isinstance(ondex, int) and os and - not (ondex >= 0 and ondex <= (64 ** os - 1))): - raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " - f"code={code}.") - - if not fs: # compute fs from index - if cs % 4: - raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise InvalidCodeSizeError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - - # both is hard code + converted index - both = (f"{code}{intToB64(index, l=ms)}" - f"{intToB64(ondex if ondex is not None else 0, l=os)}") - - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(both))) - - if (cs % 4) != ps - ls: # adjusted pad given lead bytes - raise InvalidCodeSizeError(f"Invalid code={both} for converted" - f" raw pad size={ps}.") - - n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code + index - # convert code both to right align b2 int then left shift in pad bits - # then convert to bytes - bcode = (b64ToInt(both) << (2 * (ps - ls))).to_bytes(n, 'big') - full = bcode + bytes([0] * ls) + raw - - bfs = len(full) # binary full size - if bfs % 3 or (bfs * 4 // 3) != fs: # invalid size - raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") - - return full - - - def _exfil(self, qb64b): - """ - Extracts self.code, self.index, and self.raw from qualified base64 bytes qb64b - - cs = hs + ss - ms = ss - os (main index size) - when fs None then size computed & fs = size * 4 + cs - """ - if not qb64b: # empty need more bytes - raise ShortageError("Empty material.") - - first = qb64b[:1] # extract first char code selector - if hasattr(first, "decode"): - first = first.decode("utf-8") - if first not in self.Hards: - if first[0] == '-': - raise UnexpectedCountCodeError("Unexpected count code start" - "while extracing Indexer.") - elif first[0] == '_': - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Indexer.") - else: - raise UnexpectedCodeError(f"Unsupported code start char={first}.") - - hs = self.Hards[first] # get hard code size - if len(qb64b) < hs: # need more bytes - raise ShortageError(f"Need {hs - len(qb64b)} more characters.") - - hard = qb64b[:hs] # get hard code - if hasattr(hard, "decode"): - hard = hard.decode("utf-8") - if hard not in self.Sizes: - raise UnexpectedCodeError(f"Unsupported code ={hard}.") - - hs, ss, os, fs, ls = self.Sizes[hard] # assumes hs in both tables consistent - cs = hs + ss # both hard + soft code size - ms = ss - os - # assumes that unit tests on Indexer and IndexerCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) - # assumes no variable length indexed codes so fs is not None - - if len(qb64b) < cs: # need more bytes - raise ShortageError(f"Need {cs - len(qb64b)} more characters.") - - index = qb64b[hs:hs+ms] # extract index/size chars - if hasattr(index, "decode"): - index = index.decode("utf-8") - index = b64ToInt(index) # compute int index - - ondex = qb64b[hs+ms:hs+ms+os] # extract ondex chars - if hasattr(ondex, "decode"): - ondex = ondex.decode("utf-8") - - if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 - ondex = b64ToInt(ondex) if os else None # compute ondex from code - if ondex: # not zero or None so error - raise ValueError(f"Invalid ondex={ondex} for code={hard}.") - else: - ondex = None # zero so set to None when current only - else: - ondex = b64ToInt(ondex) if os else index - - # index is index for some codes and variable length for others - if not fs: # compute fs from index which means variable length - if cs % 4: - raise ValidationError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise ValidationError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - - if len(qb64b) < fs: # need more bytes - raise ShortageError(f"Need {fs - len(qb64b)} more chars.") - - qb64b = qb64b[:fs] # fully qualified primitive code plus material - if hasattr(qb64b, "encode"): # only convert extracted chars from stream - qb64b = qb64b.encode("utf-8") - - # strip off prepended code and append pad characters - #ps = cs % 4 # pad size ps = cs mod 4, same pad chars and lead bytes - #base = ps * b'A' + qb64b[cs:] # replace prepend code with prepad zeros - #raw = decodeB64(base)[ps+ls:] # decode and strip off ps+ls prepad bytes - - # check for non-zeroed pad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero - paw = decodeB64(base) # decode base to leave prepadded raw - pi = (int.from_bytes(paw[:ps], "big")) # prepad as int - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed prepad bits = " - f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") - raw = paw[ps:] # strip off ps prepad paw bytes - else: # not ps. IF not ps THEN may or may not be ls (lead) - base = qb64b[cs:] # strip off code leaving lead chars if any and value - # decode lead chars + val leaving lead bytes + raw bytes - # then strip off ls lead bytes leaving raw - paw = decodeB64(base) # decode base to leave prepadded paw bytes - li = int.from_bytes(paw[:ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") - - raw = paw[ls:] - - if len(raw) != (len(qb64b) - cs) * 3 // 4: # exact lengths - raise ConversionError(f"Improperly qualified material = {qb64b}") - - self._code = hard - self._index = index - self._ondex = ondex - self._raw = raw # must be bytes for crpto opts and immutable not bytearray - - - - def _bexfil(self, qb2): - """ - Extracts self.code, self.index, and self.raw from qualified base2 bytes qb2 - - cs = hs + ss - ms = ss - os (main index size) - when fs None then size computed & fs = size * 4 + cs - """ - if not qb2: # empty need more bytes - raise ShortageError("Empty material, Need more bytes.") - - first = nabSextets(qb2, 1) # extract first sextet as code selector - if first not in self.Bards: - if first[0] == b'\xf8': # b64ToB2('-') - raise UnexpectedCountCodeError("Unexpected count code start" - "while extracing Matter.") - elif first[0] == b'\xfc': # b64ToB2('_') - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Matter.") - else: - raise UnexpectedCodeError(f"Unsupported code start sextet={first}.") - - hs = self.Bards[first] # get code hard size equvalent sextets - bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets - if len(qb2) < bhs: # need more bytes - raise ShortageError(f"Need {bhs - len(qb2)} more bytes.") - - hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code - if hard not in self.Sizes: - raise UnexpectedCodeError(f"Unsupported code ={hard}.") - - hs, ss, os, fs, ls = self.Sizes[hard] - cs = hs + ss # both hs and ss - ms = ss - os - # assumes that unit tests on Indexer and IndexerCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) - - bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets - if len(qb2) < bcs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - - both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - index = b64ToInt(both[hs:hs+ms]) # compute index - - if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 - ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else None # compute ondex from code - if ondex: # not zero or None so error - raise ValueError(f"Invalid ondex={ondex} for code={hard}.") - else: - ondex = None # zero so set to None when current only - else: - ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else index - - if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 - if ondex: # not zero so error - raise ValueError(f"Invalid ondex={ondex} for code={hard}.") - else: # zero so set to None - ondex = None - - if not fs: # compute fs from size chars in ss part of code - if cs % 4: - raise ValidationError(f"Whole code size not multiple of 4 for " - f"variable length material. cs={cs}.") - if os != 0: - raise ValidationError(f"Non-zero other index size for " - f"variable length material. os={os}.") - fs = (index * 4) + cs - - bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets - if len(qb2) < bfs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bfs - len(qb2))) - - qb2 = qb2[:bfs] # extract qb2 fully qualified primitive code plus material - - # check for non-zeroed prepad bits or lead bytes - ps = cs % 4 # code pad size ps = cs mod 4 - pbs = 2 * (ps if ps else ls) # pad bit size in bits - if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls - # convert last byte of code bytes in which are pad bits to int - pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) - if pi & (2 ** pbs - 1 ): # masked pad bits non-zero - raise ValueError(f"Non zeroed pad bits = " - f"{pi & (2 ** pbs - 1 ):>08b} in 0x{pi:02x}.") - else: # not ps. IF not ps THEN may or may not be ls (lead) - li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int - if li: # pre pad lead bytes must be zero - if ls == 1: - raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") - else: - raise ValueError(f"Non zeroed lead bytes = 0x{li:02x}.") - - - raw = qb2[(bcs + ls):] # strip code and leader bytes from qb2 to get raw - - if len(raw) != (len(qb2) - bcs - ls): # exact lengths - raise ConversionError(f"Improperly qualified material = {qb2}") - - self._code = hard - self._index = index - self._ondex = ondex - self._raw = bytes(raw) # must be bytes for crypto ops and not bytearray mutable - - -class Siger(Indexer): - """ - Siger is subclass of Indexer, indexed signature material, - - Adds .verfer property which is instance of Verfer that provides - associated signature verifier. - - See Indexer for inherited attributes and properties: - - Attributes: - - Properties: - verfer (Verfer): instance if any provides public verification key - - Methods: - - Hidden: - _verfer (Verfer): value for .verfer property - - - """ - - def __init__(self, verfer=None, **kwa): - """Initialze instance - - Parameters: See Matter for inherted parameters - verfer (Verfer): instance if any provides public verification key - - """ - super(Siger, self).__init__(**kwa) - if self.code not in IdxSigDex: - raise ValidationError("Invalid code = {} for Siger." - "".format(self.code)) - self.verfer = verfer - - @property - def verfer(self): - """ - Property verfer: - Returns Verfer instance - Assumes ._verfer is correctly assigned - """ - return self._verfer - - @verfer.setter - def verfer(self, verfer): - """ verfer property setter """ - self._verfer = verfer - - -@dataclass(frozen=True) -class CounterCodex: - """ - CounterCodex is codex hard (stable) part of all counter derivation codes. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - - ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. - WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. - NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. - TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. - FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. - TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. - SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegators or issuers event - TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. - SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - KERIProtocolStack: str = '--AAA' # KERI ACDC Protocol Stack CESR Version - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" - -CtrDex = CounterCodex() - - -@dataclass(frozen=True) -class ProtocolGenusCodex: - """ProtocolGenusCodex is codex of protocol genera for code table. - - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - KERI: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables - ACDC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables - - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" - # duplicate values above just result in multiple entries in tuple so - # in inclusion still works - -ProDex = ProtocolGenusCodex() # Make instance - - -@dataclass(frozen=True) -class AltCounterCodex: - """ - CounterCodex is codex hard (stable) part of all counter derivation codes. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - - ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. - WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. - NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. - TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. - FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. - TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. - SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegators or issuers event - TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. - SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event - SadPathSig: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content - SadPathSigGroup: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples - PathedMaterialQuadlets: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) - MessageDataGroups: str = '-U' # Composed Message Data Group or Primitive - AttachedMaterialQuadlets: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) - MessageDataMaterialQuadlets: str = '-W' # Composed Grouped Message Data Quadlet (4 char each) - CombinedMaterialQuadlets: str = '-X' # Combined Message Data + Attachments Quadlet (4 char each) - MaterialGroups: str = '-Y' # Composed Generic Material Group or Primitive - MaterialQuadlets: str = '-Z' # Composed Generic Material Quadlet (4 char each) - BigMessageDataGroups: str = '-0U' # Composed Message Data Group or Primitive - BigAttachedMaterialQuadlets: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) - BigMessageDataMaterialQuadlets: str = '-0W' # Composed Grouped Message Data Quadlet (4 char each) - BigCombinedMaterialQuadlets: str = '-0X' # Combined Message Data + Attachments Quadlet (4 char each) - BigMaterialGroups: str = '-0Y' # Composed Generic Material Group or Primitive - BigMaterialQuadlets: str = '-0Z' # Composed Generic Material Quadlet (4 char each) - - - def __iter__(self): - return iter(astuple(self)) # enables inclusion test with "in" - - -class Counter: - """ - Counter is fully qualified cryptographic material primitive base class for - counter primitives (framing composition grouping count codes). - - Sub classes are derivation code and key event element context specific. - - Includes the following attributes and properties: - - Attributes: - - Properties: - .code is str derivation code to indicate cypher suite - .raw is bytes crypto material only without code - .pad is int number of pad chars given raw - .count is int count of grouped following material (not part of counter) - .qb64 is str in Base64 fully qualified with derivation code + crypto mat - .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat - .qb2 is bytes in binary with derivation code + crypto material - - Hidden: - ._code is str value for .code property - ._raw is bytes value for .raw property - ._pad is method to compute .pad property - ._count is int value for .count property - ._infil is method to compute fully qualified Base64 from .raw and .code - ._exfil is method to extract .code and .raw from fully qualified Base64 - - """ - Codex = CtrDex - # Hards table maps from bytes Base64 first two code chars to int of - # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter - # is always > 0 and hs + ss = fs always - Hards = ({('-' + chr(c)): 2 for c in range(65, 65 + 26)}) - Hards.update({('-' + chr(c)): 2 for c in range(97, 97 + 26)}) - Hards.update([('-0', 3)]) - Hards.update([('--', 5)]) - # Sizes table maps hs chars of code to Sizage namedtuple of (hs, ss, fs) - # where hs is hard size, ss is soft size, and fs is full size - # soft size, ss, should always be > 0 and hs+ss=fs for Counter - Sizes = { - '-A': Sizage(hs=2, ss=2, fs=4, ls=0), - '-B': Sizage(hs=2, ss=2, fs=4, ls=0), - '-C': Sizage(hs=2, ss=2, fs=4, ls=0), - '-D': Sizage(hs=2, ss=2, fs=4, ls=0), - '-E': Sizage(hs=2, ss=2, fs=4, ls=0), - '-F': Sizage(hs=2, ss=2, fs=4, ls=0), - '-G': Sizage(hs=2, ss=2, fs=4, ls=0), - '-H': Sizage(hs=2, ss=2, fs=4, ls=0), - '-I': Sizage(hs=2, ss=2, fs=4, ls=0), - '-J': Sizage(hs=2, ss=2, fs=4, ls=0), - '-K': Sizage(hs=2, ss=2, fs=4, ls=0), - '-L': Sizage(hs=2, ss=2, fs=4, ls=0), - '-V': Sizage(hs=2, ss=2, fs=4, ls=0), - '-0V': Sizage(hs=3, ss=5, fs=8, ls=0), - '--AAA': Sizage(hs=5, ss=3, fs=8, ls=0), - } - # Bards table maps to hard size, hs, of code from bytes holding sextets - # converted from first two code char. Used for ._bexfil. - Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) - - def __init__(self, code=None, count=None, countB64=None, - qb64b=None, qb64=None, qb2=None, strip=False): - """ - Validate as fully qualified - Parameters: - code (str | None): stable (hard) part of derivation code - count (int | None): count for composition. - Count may represent quadlets/triplet, groups, primitives or - other numericy - When both count and countB64 are None then count defaults to 1 - countB64 (str | None): count for composition as Base64 - countB64 may represent quadlets/triplet, groups, primitives or - other numericy - qb64b (bytes | bytearray | None): fully qualified crypto material text domain - qb64 (str | None) fully qualified crypto material text domain - qb2 (bytes | bytearray | None) fully qualified crypto material binary domain - strip (bool): True means strip counter contents from input stream - bytearray after parsing qb64b or qb2. False means do not strip. - default False - - - Needs either code or qb64b or qb64 or qb2 - Otherwise raises EmptyMaterialError - When code and count provided then validate that code and count are correct - Else when qb64b or qb64 or qb2 provided extract and assign - .code and .count - - """ - if code is not None: # code provided - if code not in self.Sizes: - raise InvalidCodeError("Unsupported code={}.".format(code)) - - hs, ss, fs, ls = self.Sizes[code] # get sizes for code - cs = hs + ss # both hard + soft code size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - - if count is None: - count = 1 if countB64 is None else b64ToInt(countB64) - - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - self._code = code - self._count = count - - elif qb64b is not None: - self._exfil(qb64b) - if strip: # assumes bytearray - del qb64b[:self.Sizes[self.code].fs] - - elif qb64 is not None: - self._exfil(qb64) - - elif qb2 is not None: # rewrite to use direct binary exfiltration - self._bexfil(qb2) - if strip: # assumes bytearray - del qb2[:self.Sizes[self.code].fs * 3 // 4] - - else: - raise EmptyMaterialError("Improper initialization need either " - "(code and count) or qb64b or " - "qb64 or qb2.") - - @property - def code(self): - """ - Returns ._code - Makes .code read only - """ - return self._code - - - @property - def count(self): - """ - Returns ._count - Makes ._count read only - """ - return self._count - - - @property - def qb64b(self): - """ - Property qb64b: - Returns Fully Qualified Base64 Version encoded as bytes - Assumes self.raw and self.code are correctly populated - """ - return self._infil() - - - @property - def qb64(self): - """ - Property qb64: - Returns Fully Qualified Base64 Version - Assumes self.raw and self.code are correctly populated - """ - return self.qb64b.decode("utf-8") - - - @property - def qb2(self): - """ - Property qb2: - Returns Fully Qualified Binary Version Bytes - """ - return self._binfil() - - - def countToB64(self, l=None): - """ Returns count as Base64 left padded with "A"s - Parameters: - l (int | None): minimum number characters including left padding - When not provided use the softsize of .code - - """ - if l is None: - _, ss, _, _ = self.Sizes[self.code] - l = ss - return (intToB64(self.count, l=l)) - - - @staticmethod - def semVerToB64(version="", major=0, minor=0, patch=0): - """ Converts semantic version to Base64 representation of countB64 - suitable for CESR protocol genus and version - - Returns: - countB64 (str): suitable for input to Counter - example: Counter(countB64=semVerToB64(version = "1.0.0")) - - Parameters: - version (str | None): dot separated semantic version string of format - "major.minor.patch" - major (int): When version is None or empty then use major,minor, patch - minor (int): When version is None or empty then use major,minor, patch - patch (int): When version is None or empty then use major,minor, patch - - each of major, minor, patch must be in range [0,63] for represenation as - three Base64 characters - - """ - parts = [major, minor, patch] - if version: - splits = version.split(".", maxsplit=3) - splits = [(int(s) if s else 0) for s in splits] - for i in range(3-len(splits),0, -1): - splits.append(parts[-i]) - parts = splits - - for p in parts: - if p < 0 or p > 63: - raise ValueError(f"Out of bounds semantic version. " - f"Part={p} is < 0 or > 63.") - return ("".join(intToB64(p, l=1) for p in parts)) - - - def _infil(self): - """ - Returns fully qualified attached sig base64 bytes computed from - self.code and self.count. - """ - code = self.code # codex value chars hard code - count = self.count # index value int used for soft - - hs, ss, fs, ls = self.Sizes[code] - cs = hs + ss # both hard + soft size - if fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - # both is hard code + converted count - both = "{}{}".format(code, intToB64(count, l=ss)) - - # check valid pad size for whole code size - if len(both) % 4: # no pad - raise InvalidCodeSizeError("Invalid size = {} of {} not a multiple of 4." - .format(len(both), both)) - # prepending full derivation code with index and strip off trailing pad characters - return (both.encode("utf-8")) - - - def _binfil(self): - """ - Returns bytes of fully qualified base2 bytes, that is .qb2 - self.code converted to Base2 left shifted with pad bits - equivalent of Base64 decode of .qb64 into .qb2 - """ - code = self.code # codex chars hard code - count = self.count # index value int used for soft - - hs, ss, fs, ls = self.Sizes[code] - cs = hs + ss - if fs != cs or cs % 4: # fs must be cs and multiple of 4 for count codes - raise InvalidCodeSizeError("Whole code size not full size or not " - "multiple of 4. cs={} fs={}.".format(cs, fs)) - - if count < 0 or count > (64 ** ss - 1): - raise InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) - - # both is hard code + converted count - both = "{}{}".format(code, intToB64(count, l=ss)) - if len(both) != cs: - raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." - .format(cs, len(both))) - - return (codeB64ToB2(both)) # convert to b2 left shift if any - - - def _exfil(self, qb64b): - """ - Extracts self.code and self.count from qualified base64 bytes qb64b - """ - if not qb64b: # empty need more bytes - raise ShortageError("Empty material, Need more characters.") - - first = qb64b[:2] # extract first two char code selector - if hasattr(first, "decode"): - first = first.decode("utf-8") - if first not in self.Hards: - if first[0] == '_': - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Counter.") - else: - raise UnexpectedCodeError("Unsupported code start ={}.".format(first)) - - hs = self.Hards[first] # get hard code size - if len(qb64b) < hs: # need more bytes - raise ShortageError("Need {} more characters.".format(hs - len(qb64b))) - - hard = qb64b[:hs] # get hard code - if hasattr(hard, "decode"): - hard = hard.decode("utf-8") # decode converts bytearray/bytes to str - if hard not in self.Sizes: # Sizes needs str not bytes - raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - - hs, ss, fs, ls = self.Sizes[hard] # assumes hs consistent in both tables - cs = hs + ss # both hard + soft code size - - # assumes that unit tests on Counter and CounterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 - - if len(qb64b) < cs: # need more bytes - raise ShortageError("Need {} more characters.".format(cs - len(qb64b))) - - count = qb64b[hs:hs + ss] # extract count chars - if hasattr(count, "decode"): - count = count.decode("utf-8") - count = b64ToInt(count) # compute int count - - self._code = hard - self._count = count - - - def _bexfil(self, qb2): - """ - Extracts self.code and self.count from qualified base2 bytes qb2 - """ - if not qb2: # empty need more bytes - raise ShortageError("Empty material, Need more bytes.") - - first = nabSextets(qb2, 2) # extract first two sextets as code selector - if first not in self.Bards: - if first[0] == b'\xfc': # b64ToB2('_') - raise UnexpectedOpCodeError("Unexpected op code start" - "while extracing Matter.") - else: - raise UnexpectedCodeError("Unsupported code start sextet={}.".format(first)) - - hs = self.Bards[first] # get code hard size equvalent sextets - bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets - if len(qb2) < bhs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bhs - len(qb2))) - - hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code - if hard not in self.Sizes: - raise UnexpectedCodeError("Unsupported code ={}.".format(hard)) - - hs, ss, fs, ls = self.Sizes[hard] - cs = hs + ss # both hs and ss - # assumes that unit tests on Counter and CounterCodex ensure that - # .Codes and .Sizes are well formed. - # hs consistent and hs > 0 and ss > 0 and fs = hs + ss and not fs % 4 - - bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets - if len(qb2) < bcs: # need more bytes - raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) - - both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code - count = b64ToInt(both[hs:hs + ss]) # get count - - self._code = hard - self._count = count - - -class Sadder: - """ - Sadder is self addressed data (SAD) serializer-deserializer class - - Instance creation of a Sadder does not verifiy it .said property it merely - extracts it. In order to ensure Sadder instance has a verified .said then - must call .saider.verify(sad=self.ked) - - Has the following public properties: - - Properties: - raw (bytes): of serialized event only - ked (dict): self addressed data dict - kind (str): serialization kind coring.Serials such as JSON, CBOR, MGPK, CESR - size (int): number of bytes in serialization - version (Versionage): protocol version (Major, Minor) - proto (str): Protocolage value as protocol identifier such as KERI, ACDC - label (str): Saidage value as said field label - saider (Saider): of SAID of this SAD .ked['d'] if present - said (str): SAID of .saider qb64 - saidb (bytes): SAID of .saider qb64b - pretty (str): Pretty JSON of this SAD - - Hidden Attributes: - ._raw is bytes of serialized event only - ._ked is key event dict - ._kind is serialization kind string value (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - ._size is int of number of bytes in serialed event only - ._version is Versionage instance of event version - ._proto (str): Protocolage value as protocol type identifier - ._saider (Saider): instance for this Sadder's SAID - - Note: - loads and jumps of json use str whereas cbor and msgpack use bytes - - """ - - def __init__(self, raw=b'', ked=None, sad=None, kind=None, saidify=False, - code=MtrDex.Blake3_256): - """ - Deserialize if raw provided does not verify assumes embedded said is valid - Serialize if ked provided but not raw verifies if verify is True? - When serializing if kind provided then use kind instead of field in ked - - Parameters: - raw (bytes): serialized event - ked is key event dict or None - if None its deserialized from raw - kind is serialization kind string value or None (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - if kind is None then its extracted from ked or raw - saidify (bool): True means compute said for ked - code is .diger default digest code for computing said .saider - - """ - self._code = code # need default code for .saider - if raw: # deserialize raw using property setter - self.raw = raw # raw property setter does the deserialization - elif ked: # serialize ked using property setter - #ToDo when pass in ked and saidify True then compute said - self._kind = kind - self.ked = ked # ked property setter does the serialization - elif sad: - # ToDo do we need this or should we be using ked above with saidify flag - self._clone(sad=sad) # copy fields from sad - else: - raise ValueError("Improper initialization need sad, raw or ked.") - - - def _clone(self, sad): - """ copy hidden attributes from sad """ - self._raw = sad.raw - self._ked = sad.ked - self._kind = sad.kind - self._size = sad.size - self._version = sad.version - self._proto = sad.proto - self._saider = sad.saider - - - def _inhale(self, raw): - """ - Parses serilized event ser of serialization kind and assigns to - instance attributes. - - Parameters: - raw is bytes of serialized event - kind is str of raw serialization kind (see namedtuple Serials) - size is int size of raw to be deserialized - - Note: - loads and jumps of json use str whereas cbor and msgpack use bytes + kind (str): serialization algorithm of sad, one of Serials + used to override that given by 'v' field if any in sad + otherwise default is Serials.json """ - proto, kind, version, size = sniff(raw) - if version != Version: - raise VersionError("Unsupported version = {}.{}, expected {}." - "".format(version.major, version.minor, Version)) - if len(raw) < size: - raise ShortageError("Need more bytes.") - - ked = loads(raw=raw, size=size, kind=kind) + knd = Kinds.json + if 'v' in sad: # versioned sad + _, _, knd, _, _ = deversify(sad['v']) - return ked, proto, kind, version, size + if not kind: # match logic of Serder for kind + kind = knd + return dumps(sad, kind=kind) - def _exhale(self, ked, kind=None): + @classmethod + def saidify(clas, + sad: dict, + *, + code: str = MtrDex.Blake3_256, + kind: str = None, + label: str = Saids.d, + ignore: list = None, **kwa): """ - Returns sizeify(ked, kind) + Derives said from sad and injects it into copy of sad and said and + injected sad - From sizeify - Returns tuple of (raw, proto, kind, ked, version) where: - raw (str): serialized event as bytes of kind - proto (str): protocol type as value of Protocolage - kind (str): serialzation kind as value of Serialage - ked (dict): key event dict or sad dict - version (Versionage): instance + Returns: + result (tuple): of the form (saider, sad) where saider is Saider + instance generated from sad using code and sad is copy of + parameter sad but with its label id field filled + in with generated said from saider Parameters: - ked (dict): key event dict or sad dict - kind (str): value of Serials serialization kind. - When not provided use + sad (dict): serializable dict + code (str): digest type code from DigDex + kind (str): serialization algorithm of sad, one of Serials + used to override that given by 'v' field if any in sad + otherwise default is Serials.json + label (str): Saidage value as said field label in which to inject said + ignore (list): fields to ignore when generating SAID - Assumes only supports Version """ - return sizeify(ked=ked, kind=kind) + if label not in sad: + raise KeyError("Missing id field labeled={} in sad.".format(label)) + raw, sad = clas._derive(sad=sad, code=code, kind=kind, label=label, ignore=ignore) + saider = clas(raw=raw, code=code, kind=kind, label=label, ignore=ignore, **kwa) + sad[label] = saider.qb64 + return saider, sad - def compare(self, said=None): + @classmethod + def _derive(clas, sad: dict, *, + code: str = MtrDex.Blake3_256, + kind: str = None, + label: str = Saids.d, + ignore: list = None): """ - Returns True if said and either .saider.qb64 or .saider.qb64b match - via string equality == + Derives raw said from sad with .Dummy filled sad[label] - Convenience method to allow comparison of own .saider digest self.raw - with some other purported said of self.raw + Returns: + raw (bytes): raw said from sad with dummy filled label id field Parameters: - said is qb64b or qb64 SAID of ser to compare with .said + sad (dict): self addressed data to be injected with dummy and serialized + code (str): digest type code from DigDex + kind (str): serialization algorithm of sad, one of Serials + used to override that given by 'v' field if any in sad + otherwise default is Serials.json + label (str): Saidage value as said field label in which to inject dummy + ignore (list): fields to ignore when generating SAID """ + if code not in DigDex: + raise ValueError(f"Unsupported digest {code=}.") - if said is not None: - if hasattr(said, "encode"): - said = said.encode('utf-8') # makes bytes - - return said == self.saidb # matching - - else: - raise ValueError("Both said and saider may not be None.") - - - @property - def raw(self): - """ raw property getter """ - return self._raw - - @raw.setter - def raw(self, raw): - """ raw property setter """ - ked, proto, kind, version, size = self._inhale(raw=raw) - self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray - self._ked = ked - self._proto = proto - self._kind = kind - self._version = version - self._size = size - self._saider = Saider(qb64=ked["d"], code=self._code) - - @property - def ked(self): - """ ked property getter""" - return self._ked - - @ked.setter - def ked(self, ked): - """ ked property setter assumes ._kind """ - raw, proto, kind, ked, version = self._exhale(ked=ked, kind=self._kind) - size = len(raw) - self._raw = raw[:size] - self._ked = ked - self._proto = proto - self._kind = kind - self._size = size - self._version = version - self._saider = Saider(qb64=ked["d"], code=self._code) - - @property - def kind(self): - """ kind property getter""" - return self._kind - - @kind.setter - def kind(self, kind): - """ kind property setter Assumes ._ked. Serialization kind. """ - raw, proto, kind, ked, version = self._exhale(ked=self._ked, kind=kind) - size = len(raw) - self._raw = raw[:size] - self._proto = proto - self._ked = ked - self._kind = kind - self._size = size - self._version = version - self._saider = Saider(qb64=ked["d"], code=self._code) + sad = dict(sad) # make shallow copy so don't clobber original sad + # fill id field denoted by label with dummy chars to get size correct + sad[label] = clas.Dummy * Matter.Sizes[code].fs + if 'v' in sad: # if versioned then need to set size in version string + raw, proto, kind, sad, version = sizeify(ked=sad, kind=kind) + ser = dict(sad) + if ignore: # delete ignore fields in said calculation from ser dict + for f in ignore: + del ser[f] - @property - def size(self): - """ size property getter""" - return self._size + cpa = clas._serialize(ser, kind=kind) # serialize ser + return (Diger._digest(ser=cpa, code=code), sad) # raw digest and sad - @property - def version(self): + def derive(self, sad, code=None, **kwa): """ - version property getter - Returns: - (Versionage): - """ - return self._version - - - @property - def proto(self): - """ proto property getter - protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' + result (tuple): (raw, sad) raw said as derived from serialized dict + and modified sad during derivation. - Returns: - (str): Protocolage value as protocol type + Parameters: + sad (dict): self addressed data to be serialized + code (str): digest type code from DigDex. + kind (str): serialization algorithm of sad, one of Serials + used to override that given by 'v' field if any in sad + otherwise default is Serials.json + label (str): Saidage value of said field labelin which to inject dummy """ - return self._proto + code = code if code is not None else self.code + return self._derive(sad=sad, code=code, **kwa) - @property - def saider(self): - """ - Returns Diger of digest of self.raw - diger (digest material) property getter + def verify(self, sad, *, prefixed=False, versioned=True, code=None, + kind=None, label=Saids.d, ignore=None, **kwa): """ - return self._saider + Returns: + result (bool): True means derivation from sad with dummy label + field value replacement for ._code matches .qb64. False otherwise + If prefixed is True then also validates that label field of + provided sad also matches .qb64. False otherwise + If versioned is True and provided sad includes version field 'v' + then also validates that version field 'v' of provided + sad matches the version field of modified sad that results from + the derivation process. The size chars in the version field + are set to the size of the sad during derivation. False otherwise. - @property - def said(self): - """ - Returns str qb64 of .ked["d"] (said when ked is SAD) - said (self-addressing identifier) property getter + Parameters: + sad (dict): self addressed data to be serialized + prefixed (bool): True means also verify if labeled field in + sad matches own .qb64 + versioned (bool): + code (str): digest type code from DigDex. + kind (str): serialization algorithm of sad, one of Serials + used to override that given by 'v' field if any in sad + otherwise default is Serials.json + label (str): Saidage value of said field label in which to inject dummy + ignore (list): fields to ignore when generating SAID """ - return self.saider.qb64 + try: + # override ensure code is self.code + raw, dsad = self._derive(sad=sad, code=self.code, kind=kind, label=label, ignore=ignore) + saider = Saider(raw=raw, code=self.code, ignore=ignore, **kwa) + if self.qb64b != saider.qb64b: + return False # not match .qb64b - @property - def saidb(self): - """ - Returns bytes qb64b of .ked["d"] (said when ked is SAD) - said (self-addressing identifier) property getter - """ - return self.saider.qb64b + if 'v' in sad and versioned: + if sad['v'] != dsad['v']: + return False # version fields not match - def pretty(self, *, size=1024): - """ - Returns str JSON of .ked with pretty formatting + if prefixed and sad[label] != self.qb64: # check label field + return False # label id field not match .qb64 - ToDo: add default size limit on pretty when used for syslog UDP MCU - like 1024 for ogler.logger - """ - return json.dumps(self.ked, indent=1)[:size if size is not None else None] + except Exception as ex: + return False + + return True @@ -5386,18 +3664,26 @@ class Tholder: when unweighted is size of int thold since don't have anyway to know size of keys list in this case - .limen is qualified b64 signing threshold suitable for CESR serialization. + .limen is qualified b64b signing threshold suitable for CESR serialization. either Number.qb64b or Bexter.qb64b. The b64 portion of limen with code stripped (Bexter.bext) of [["1/2", "1/2", "1/4", "1/4", "1/4"], ["1", "1"]] is '1s2c1s2c1s4c1s4c1s4a1c1' basically slash is 's', comma is 'c', - and ANDed clauses are delimited by 'a'. + ANDed clauses are delimited by 'a'. + Each clause top level weight may be optionally a weighted set of weights + delimited by 'k' for the weight on the set and 'v' for the weights in + the set. + [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + .sith is original signing threshold suitable for value to be serialized as json, cbor, mgpk in key event message as either: non-negative hex number str or list of str rational number fractions >= 0 and <= 1 or list of list of str rational number fractions >= 0 and <= 1 + list of list of weighted map of weights .thold is parsed signing threshold suitable for calculating satisfaction. either as int or list of Fractions @@ -5405,8 +3691,8 @@ class Tholder: .num is int signing threshold when not ._weighted Methods: - .satisfy returns bool, True means ilist of verified signature key indices satisfies - threshold, False otherwise. + .satisfy returns bool, True means list of verified signature key indices + satisfies the threshold, False otherwise. Static Methods: weight (str): converts weight str expression into either int or Fraction @@ -5433,42 +3719,50 @@ def __init__(self, *, thold=None , limen=None, sith=None, **kwa): Accepts signing threshold in various forms so that may output correct forms for serialization and/or calculation of satisfaction. - Parameters: - sith is signing threshold (current or next) expressed as either: - non-negative int of threshold number (M-of-N threshold) - next threshold may be zero - non-negative hex string of threshold number (M-of-N threshold) - next threshold may be zero - fractional weight clauses which may be expressed as either: - an iterable of rational number fraction strings >= 0 and <= 1 - an iterable of iterables of rational number fraction strings >= 0 and <= 1 - JSON serialized str of either: - list of rational number fraction strings >= 0 and <= 1 or - list of list of rational number fraction strings >= 0 and <= 1 + The thold representation is meant to accept thresholds from computable + expressions for satisfaction of a threshold + The limen representation is meant to parse threshold expressions from + CESR serializations of key event message fields or attachments. - limen is qualified signing threshold (current or next) expressed as either: - Number.qb64 or .qb64b of integer threshold or - Bexter.qb64 or .qb64b of fractional weight clauses which may be either: - Base64 delimited clauses of fractions - Base64 delimited clauses of fractions + The sith representation is meant to parse threhold expressions from + deserializations of JSON, CBOR, or MGPK key event message fields or + the command line or configuration files. + + + Parameters: thold is signing threshold (current or next) is suitable for computing the satisfaction of a threshold and is expressed as either: int of threshold number (M of N) fractional weight clauses which may be expressed as either: - an iterable of Fractions or - an iterable of iterables of Fractions. + sequence of either Fractions or tuples of Fraction and + sequence of Fractions + sequence of sequence of either Fractions or tuples of + Fraction and sequence of Fractions - The sith representation is meant to parse threhold expressions from - deserializations of JSON, CBOR, or MGPK key event message fields or - the command line or configuration files. - - The limen representation is meant to parse threshold expressions from - CESR serializations of key event message fields or attachments. + limen is qualified signing threshold (current or next) expressed as either: + Number.qb64 or .qb64b of integer threshold or + Bexter.qb64 or .qb64b of fractional weight clauses which may be either: + Base64 delimited clauses of fractions + Base64 delimited clauses of fractions - The thold representation is meant to accept thresholds from computable - expressions for satisfaction of a threshold + sith is signing threshold (current or next) expressed as either: + non-negative int of threshold number (M-of-N threshold) + next threshold may be zero + non-negative hex string of threshold number (M-of-N threshold) + next threshold may be zero + fractional weight clauses which may be expressed as either: + sequence of rational number fraction strings >= 0 and <= 1 + sequence of either rational number fraction strings >= 0 and <= 1 or + map with key rational number string and value as sequence + of rational number fraction strings + rational number fraction string + sequence of sequences of rational number fraction strings >= 0 and <= 1 + sequence of sequnces of either rational number fraction strings or + map with key rational number fraction string with value sequence of + rationaly number fraction strings + JSON serialized str of the above: """ @@ -5513,9 +3807,24 @@ def sith(self): """ sith property getter """ # make sith expression of thold if self.weighted: - sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" - for f in clause] - for clause in self.thold] + sith = [] + for c in self.thold: + clause = [] + for e in c: + if isinstance(e, tuple): + f = e[0] + k = f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + v = [f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + for f in e[1]] + clause.append({k: v}) + else: + f = e + clause.append(f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}") + sith.append(clause) + + #sith = [[f"{f.numerator}/{f.denominator}" if (0 < f < 1) else f"{int(f)}" + #for f in clause] + #for clause in self.thold] if len(sith) == 1: sith = sith[0] # simplify list of one clause to clause else: @@ -5539,11 +3848,11 @@ def num(self): - def _processThold(self, thold: int | Iterable): + def _processThold(self, thold: int | Sequence): """Process thold input Parameters: - thold (int | Iterable): computable thold expression + thold (int | Sequence): computable thold expression """ if isinstance(thold, int): self._processUnweighted(thold=thold) @@ -5568,15 +3877,27 @@ def _processLimen(self, limen: str | bytes, **kwa): bexter = Bexter(raw=matter.raw, code=matter.code, **kwa) t = bexter.bext.replace('s', '/') # get clauses - thold = [clause.split('c') for clause in t.split('a')] - thold = [[self.weight(w) for w in clause] for clause in thold] + clauses = [clause.split('c') for clause in t.split('a')] + + thold = [] + for c in clauses: + clause = [] + for e in c: + k, s, v = e.partition("k") + if s: #not empty + clause.append((self.weight(k), [self.weight(w) for w in v.split("v")])) + else: + clause.append(self.weight(k)) + + thold.append(clause) + self._processWeighted(thold=thold) else: raise InvalidCodeError(f"Invalid code for limen = {matter.code}.") - def _processSith(self, sith: int | str | Iterable): + def _processSith(self, sith: int | str | Sequence): """ Process attributes for fractionall weighted threshold sith @@ -5587,9 +3908,9 @@ def _processSith(self, sith: int | str | Iterable): non-negative hex string of threshold number (M-of-N threshold) next threshold may be zero fractional weight clauses which may be expressed as either: - an iterable of rational number fraction weight str or int str + an sequence of rational number fraction weight str or int str each denoted w where 0 <= w <= 1 - an iterable of iterables of rational number fraction weight + an sequence of sequences of rational number fraction weight or int str each denoted w where 0 <= w <= 1>= 0 JSON serialized str of either: @@ -5598,194 +3919,509 @@ def _processSith(self, sith: int | str | Iterable): list of lists of rational number fraction weight strings each denoted w where 0 <= w <= 1 - when any w is 0 or 1 then representation is 0 or 1 not 0/1 or 1/1 - """ - if isinstance(sith, int): - self._processUnweighted(thold=sith) + when any w is 0 or 1 then representation is 0 or 1 not 0/1 or 1/1 + """ + if isinstance(sith, int): + self._processUnweighted(thold=sith) + + elif isinstance(sith, str) and '[' not in sith: + self._processUnweighted(thold=int(sith, 16)) + + else: # assumes sequence of weights or sequence of sequence of weights + if isinstance(sith, str): # json of weighted sith from cli + sith = json.loads(sith) # deserialize + + if not sith: # empty or None + raise ValueError(f"Empty weight list = {sith}.") + + # is it non str sequence of sequences? or non str sequnce of strs? + # must test for emply mask because all([]) == True + mask = [nonStringSequence(c) for c in sith] # check each element + if mask and not all(mask): # not empty and not sequence of sequenes + sith = [sith] # attempt to make sequnce of sequqnces of strs + + for c in sith: # get each clause + # each element of a clause must be a str or dict + mask = [(isinstance(w, str) or isinstance(w, Mapping)) for w in c] + if mask and not all(mask): # not empty and not sequence of str or dicts + raise ValueError(f"Invalid sith = {sith} some weights in" + f"clause {c} are non string.") + + # replace weight str expression, int str or fractional strings with + # int or fraction as appropriate. + thold = [] + for c in sith: # convert string fractions to Fractions + # append list of where each element is either bare weight or + # single key map with value as list of weights + # each weight is converted from its str expression + clause = [] + for e in c: # each element of clause c + if isinstance(e, Mapping): + if len(e) != 1: + raise ValueError(f"Invalid sith = {sith} nested " + f"weight map {e} in clause {c} " + f" not single key value.") + k = list(e)[0] # zeroth key is used + # convert to tuple of (weight, [list of weights]) + clause.append((self.weight(k), [self.weight(w) for w in e[k]])) + else: + clause.append(self.weight(e)) + + thold.append(clause) + + self._processWeighted(thold=thold) + + + def _processUnweighted(self, thold=0): + """ + Process attributes for unweighted (numeric) threshold thold + + Parameters: + thold (int): non-negative threshold number M-of-N threshold + + """ + if thold < 0: + raise ValueError(f"Non-positive int threshold = {thold}.") + self._thold = thold + self._weighted = False + self._size = self._thold # used to verify that keys list size is at least size + self._satisfy = self._satisfy_numeric + self._number = Number(num=thold) + self._bexter = None + + + def _processWeighted(self, thold=[]): + """ + Process attributes for fractionall weighted threshold thold + + Parameters: + thold (iterable): iterable or iterable or iterables of + rational number fraction strings >= 0 and <= 1 + + """ + for clause in thold: # sum of top level weights in clause must be >= 1 + # When element is dict then sum of value's weights must be >= 1 + top = [] # top level weights + for e in clause: + if isinstance(e, tuple): + top.append(e[0]) + if not (sum(e[1]) >= 1): + raise ValueError(f"Invalid sith clause = {clause}, " + f"element = {e}. All nested clause " + f"weight sums must be >= 1.") + else: + top.append(e) + if not (sum(top) >= 1): + raise ValueError(f"Invalid sith clause = {clause}, all top level" + f"clause weight sums must be >= 1.") + + self._thold = thold + self._weighted = True + #self._size = sum(len(clause) for clause in thold) + s = 0 + for clause in thold: + for e in clause: + if isinstance(e, tuple): + s += len(e[1]) + else: + s += 1 + self._size = s + + self._satisfy = self._satisfy_weighted + # make bext str of thold for .bexter for limen + ta = [] # list of list of fractions and/or single element map of fractions + for c in thold: + bc = [] # list of fractions and/or single element map of fractions + for e in c: + if isinstance(e, tuple): + f = e[0] + k = f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" + v = "v".join([f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" for f in e[1]]) + kv = "k".join([k, v]) + bc.append(kv) + else: + bc.append(f"{e.numerator}s{e.denominator}" if (0 < e < 1) else f"{int(e)}") + + ta.append(bc) + + bext = "a".join(["c".join(bc) for bc in ta]) + self._number = None + self._bexter = Bexter(bext=bext) + + + @staticmethod + def weight(w: str) -> Fraction: + """Returns valid weight from w else raises error (ValueError or TypeError). + w expression must evaluate to 0, 1, or strict proper rational fraction. + w expression must be 0 <= w <= 1 Else raises ValueError + w must not be float else raises TypeError + When not int w must be ratio of integers n/d else raise ValueError. + + Parameters: + w (str): threshold weight expression + """ + try: # float str or ratio str raises ValueError + if int(float(w)) != float(w): # float str + raise TypeError("Invalid weight str got float w={w}.") + w = int(w) # expression is int str + except TypeError as ex: + raise ValueError(str(ex)) from ex + + except ValueError as ex: # not float str or int str so try ration str + w = Fraction(w) + + if not 0 <= w <= 1: + raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") + return w + + + def satisfy(self, indices): + """ + Returns True if indices list of verified signature key indices satisfies + threshold, False otherwise. + + Parameters: + indices is list of non-negative indices (offsets into key list) + of verified signatures. the indices may be in any order, they + are normalized herein + """ + return (self._satisfy(indices=indices)) + + + def _satisfy_numeric(self, indices): + """ + Returns True if satisfies numeric threshold False otherwise + + Parameters: + indices is list of indices (offsets into key list) of verified signatures + """ + try: + if self.thold > 0 and len(indices) >= self.thold: # at least one + return True + + except Exception as ex: + return False + + return False + + + def _satisfy_weighted(self, indices): + """ + Returns True if satifies fractional weighted threshold False otherwise + + + Parameters: + indices is list of non-negative indices (offsets into key list) + of verified signatures. the indices may be in any order, they + are normalized herein + + """ + try: + if not indices: # empty indices + return False + + # remove duplicates with set, sort low to high + indices = sorted(set(indices)) + sats = [False] * self.size # default all satifactions to False + for idx in indices: + sats[idx] = True # set verified signature index to True + + wio = 0 # weight index offset + for clause in self.thold: + cw = 0 # init clause weight + for e in clause: + if isinstance(e, tuple): + vw = 0 # init element value weight + for w in e[1]: # sum weights of value + if sats[wio]: + vw += w + wio += 1 + if vw >= 1: # element true + cw += e[0] # add element key weight to clause weight + else: + w = e + if sats[wio]: # verified signature so weight applies + cw += w + wio += 1 + if cw < 1: # each clause must sum to at least 1 + return False + + return True # all clauses have cw >= 1 including final one, AND true + + except Exception as ex: + return False + + return False + + + +class Sadder: + """ + Sadder is self addressed data (SAD) serializer-deserializer class + + Instance creation of a Sadder does not verifiy it .said property it merely + extracts it. In order to ensure Sadder instance has a verified .said then + must call .saider.verify(sad=self.ked) + + Has the following public properties: + + Properties: + raw (bytes): of serialized event only + ked (dict): self addressed data dict + kind (str): serialization kind coring.Serials such as JSON, CBOR, MGPK, CESR + size (int): number of bytes in serialization + version (Versionage): protocol version (Major, Minor) + proto (str): Protocolage value as protocol identifier such as KERI, ACDC + label (str): Saidage value as said field label + saider (Saider): of SAID of this SAD .ked['d'] if present + said (str): SAID of .saider qb64 + saidb (bytes): SAID of .saider qb64b + pretty (str): Pretty JSON of this SAD + + Hidden Attributes: + ._raw is bytes of serialized event only + ._ked is key event dict + ._kind is serialization kind string value (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + ._size is int of number of bytes in serialed event only + ._version is Versionage instance of event version + ._proto (str): Protocolage value as protocol type identifier + ._saider (Saider): instance for this Sadder's SAID + + Note: + loads and jumps of json use str whereas cbor and msgpack use bytes + + """ + MaxVSOffset = 12 + SmellSize = MaxVSOffset + MAXVERFULLSPAN # min buffer size to inhale + + def __init__(self, raw=b'', ked=None, sad=None, kind=None, saidify=False, + code=MtrDex.Blake3_256): + """ + Deserialize if raw provided does not verify assumes embedded said is valid + Serialize if ked provided but not raw verifies if verify is True? + When serializing if kind provided then use kind instead of field in ked + + Parameters: + raw (bytes): serialized event + ked is key event dict or None + if None its deserialized from raw + kind is serialization kind string value or None (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + if kind is None then its extracted from ked or raw + saidify (bool): True means compute said for ked + code is .diger default digest code for computing said .saider + + """ + self._code = code # need default code for .saider + if raw: # deserialize raw using property setter + self.raw = raw # raw property setter does the deserialization + elif ked: # serialize ked using property setter + #ToDo when pass in ked and saidify True then compute said + self._kind = kind + self.ked = ked # ked property setter does the serialization + elif sad: + # ToDo do we need this or should we be using ked above with saidify flag + self._clone(sad=sad) # copy fields from sad + else: + raise ValueError("Improper initialization need sad, raw or ked.") + - elif isinstance(sith, str) and '[' not in sith: - self._processUnweighted(thold=int(sith, 16)) + def _clone(self, sad): + """ copy hidden attributes from sad """ + self._raw = sad.raw + self._ked = sad.ked + self._kind = sad.kind + self._size = sad.size + self._version = sad.version + self._proto = sad.proto + self._saider = sad.saider - else: # assumes iterable of weights or iterable of iterables of weights - if isinstance(sith, str): # json of weighted sith from cli - sith = json.loads(sith) # deserialize - if not sith: # empty iterable - raise ValueError(f"Empty weight list = {sith}.") + def _inhale(self, raw): + """ + Parses serilized event ser of serialization kind and assigns to + instance attributes. - # because all([]) == True have to also test for emply mask - # is it non str iterable of non str iterable of strs - mask = [nonStringIterable(c) for c in sith] - if mask and not all(mask): # not empty and not iterable of iterables - sith = [sith] # attempt to make Iterable of Iterables + Parameters: + raw is bytes of serialized event + kind is str of raw serialization kind (see namedtuple Serials) + size is int size of raw to be deserialized - for c in sith: # get each clause - mask = [isinstance(w, str) for w in c] # must be all strs - if mask and not all(mask): # not empty and not iterable of strs? - raise ValueError(f"Invalid sith = {sith} some weights in" - f"clause {c} are non string.") + Note: + loads and jumps of json use str whereas cbor and msgpack use bytes + """ + proto, vrsn, kind, size, _ = smell(raw) + if vrsn != Version: + raise VersionError("Unsupported version = {}.{}, expected {}." + "".format(vrsn.major, vrsn.minor, Version)) - # replace weight str expression, int str or fractional strings with - # int or fraction as appropriate. - thold = [] - for clause in sith: # convert string fractions to Fractions - # append list of weights converted fromnn str expression - thold.append([self.weight(w) for w in clause]) + ked = loads(raw=raw, size=size, kind=kind) - self._processWeighted(thold=thold) + return ked, proto, kind, vrsn, size - def _processUnweighted(self, thold=0): + def _exhale(self, ked, kind=None): """ - Process attributes for unweighted (numeric) threshold thold + Returns sizeify(ked, kind) + + From sizeify + Returns tuple of (raw, proto, kind, ked, version) where: + raw (str): serialized event as bytes of kind + proto (str): protocol type as value of Protocolage + kind (str): serialzation kind as value of Serialage + ked (dict): key event dict or sad dict + version (Versionage): instance Parameters: - thold (int): non-negative threshold number M-of-N threshold + ked (dict): key event dict or sad dict + kind (str): value of Serials serialization kind. + When not provided use + Assumes only supports Version """ - if thold < 0: - raise ValueError(f"Non-positive int threshold = {thold}.") - self._thold = thold - self._weighted = False - self._size = self._thold # used to verify that keys list size is at least size - self._satisfy = self._satisfy_numeric - self._number = Number(num=thold) - self._bexter = None + return sizeify(ked=ked, kind=kind) - def _processWeighted(self, thold=[]): + def compare(self, said=None): """ - Process attributes for fractionall weighted threshold thold + Returns True if said and either .saider.qb64 or .saider.qb64b match + via string equality == + + Convenience method to allow comparison of own .saider digest self.raw + with some other purported said of self.raw Parameters: - thold (iterable): iterable or iterable or iterables of - rational number fraction strings >= 0 and <= 1 + said is qb64b or qb64 SAID of ser to compare with .said """ - for clause in thold: # sum of fractions in clause must be >= 1 - if not (sum(clause) >= 1): - raise ValueError(f"Invalid sith clause = {thold}, all " - f"clause weight sums must be >= 1.") - self._thold = thold - self._weighted = True - self._size = sum(len(clause) for clause in thold) - self._satisfy = self._satisfy_weighted - # make bext str of thold for .bexter for limen - bext = [[f"{f.numerator}s{f.denominator}" if (0 < f < 1) else f"{int(f)}" - for f in clause] - for clause in thold] - bext = "a".join(["c".join(clause) for clause in bext]) - self._number = None - self._bexter = Bexter(bext=bext) + if said is not None: + if hasattr(said, "encode"): + said = said.encode('utf-8') # makes bytes + return said == self.saidb # matching - @staticmethod - def _oldcheckWeight(w: Fraction) -> Fraction: - """Returns w if 0 <= w <= 1 Else raises ValueError + else: + raise ValueError("Both said and saider may not be None.") - Parameters: - w (Fraction): Threshold weight Fraction - """ - if not 0 <= w <= 1: - raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") - return w + @property + def raw(self): + """ raw property getter """ + return self._raw - @staticmethod - def weight(w: str) -> Fraction: - """Returns valid weight from w else raises error (ValueError or TypeError). - w expression must evaluate to 0, 1, or strict proper rational fraction. - w expression must be 0 <= w <= 1 Else raises ValueError - w must not be float else raises TypeError - When not int w must be ratio of integers n/d else raise ValueError. + @raw.setter + def raw(self, raw): + """ raw property setter """ + ked, proto, kind, version, size = self._inhale(raw=raw) + self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray + self._ked = ked + self._proto = proto + self._kind = kind + self._version = version + self._size = size + self._saider = Saider(qb64=ked["d"], code=self._code) - Parameters: - w (str): threshold weight expression - """ - try: # float str or ratio str raises ValueError - if int(float(w)) != float(w): # float str - raise TypeError("Invalid weight str got float w={w}.") - w = int(w) # expression is int str - except TypeError as ex: - raise ValueError(str(ex)) from ex + @property + def ked(self): + """ ked property getter""" + return self._ked - except ValueError as ex: # not float str or int str so try ration str - w = Fraction(w) + @ked.setter + def ked(self, ked): + """ ked property setter assumes ._kind """ + raw, proto, kind, ked, version = self._exhale(ked=ked, kind=self._kind) + size = len(raw) + self._raw = raw[:size] + self._ked = ked + self._proto = proto + self._kind = kind + self._size = size + self._version = version + self._saider = Saider(qb64=ked["d"], code=self._code) - if not 0 <= w <= 1: - raise ValueError(f"Invalid weight not 0 <= {w} <= 1.") - return w + @property + def kind(self): + """ kind property getter""" + return self._kind + @kind.setter + def kind(self, kind): + """ kind property setter Assumes ._ked. Serialization kind. """ + raw, proto, kind, ked, version = self._exhale(ked=self._ked, kind=kind) + size = len(raw) + self._raw = raw[:size] + self._proto = proto + self._ked = ked + self._kind = kind + self._size = size + self._version = version + self._saider = Saider(qb64=ked["d"], code=self._code) - def satisfy(self, indices): - """ - Returns True if indices list of verified signature key indices satisfies - threshold, False otherwise. - Parameters: - indices is list of non-negative indices (offsets into key list) - of verified signatures. the indices may be in any order, they - are normalized herein - """ - return (self._satisfy(indices=indices)) + @property + def size(self): + """ size property getter""" + return self._size - def _satisfy_numeric(self, indices): + @property + def version(self): """ - Returns True if satisfies numeric threshold False otherwise + version property getter - Parameters: - indices is list of indices (offsets into key list) of verified signatures + Returns: + (Versionage): """ - try: - if self.thold > 0 and len(indices) >= self.thold: # at least one - return True - - except Exception as ex: - return False + return self._version - return False + @property + def proto(self): + """ proto property getter + protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' - def _satisfy_weighted(self, indices): + Returns: + (str): Protocolage value as protocol type """ - Returns True if satifies fractional weighted threshold False otherwise - + return self._proto - Parameters: - indices is list of non-negative indices (offsets into key list) - of verified signatures. the indices may be in any order, they - are normalized herein + @property + def saider(self): """ - try: - if not indices: # empty indices - return False + Returns Diger of digest of self.raw + diger (digest material) property getter + """ + return self._saider - # remove duplicates with set, sort low to high - indices = sorted(set(indices)) - sats = [False] * self.size # default all satifactions to False - for idx in indices: - sats[idx] = True # set verified signature index to True + @property + def said(self): + """ + Returns str qb64 of .ked["d"] (said when ked is SAD) + said (self-addressing identifier) property getter + """ + return self.saider.qb64 - wio = 0 # weight index offset - for clause in self.thold: - cw = 0 # init clause weight - for w in clause: - if sats[wio]: # verified signature so weight applies - cw += w - wio += 1 - if cw < 1: # each clause must sum to at least 1 - return False + @property + def saidb(self): + """ + Returns bytes qb64b of .ked["d"] (said when ked is SAD) + said (self-addressing identifier) property getter + """ + return self.saider.qb64b - return True # all clauses including final one cw >= 1 + def pretty(self, *, size=1024): + """ + Returns str JSON of .ked with pretty formatting - except Exception as ex: - return False + ToDo: add default size limit on pretty when used for syslog UDP MCU + like 1024 for ogler.logger + """ + return json.dumps(self.ked, indent=1)[:size if size is not None else None] - return False @@ -5867,12 +4503,3 @@ def pretty(self, *, size=1024): return json.dumps(self.pad, indent=1)[:size if size is not None else None] -def randomNonce(): - """ Generate a random ed25519 seed and encode as qb64 - - Returns: - str: qb64 encoded ed25519 random seed - """ - preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seedqb64 = Matter(raw=preseed, code=MtrDex.Ed25519_Seed).qb64 - return seedqb64 diff --git a/src/keri/core/counting.py b/src/keri/core/counting.py new file mode 100644 index 000000000..4fc8774df --- /dev/null +++ b/src/keri/core/counting.py @@ -0,0 +1,894 @@ +# -*- coding: utf-8 -*- +""" +keri.core.counting module + +Provides versioning support for Counter classes and codes +""" +import copy + +from dataclasses import dataclass, astuple, asdict +from collections import namedtuple + +from ..help import helping +from ..help.helping import sceil +from ..help.helping import (intToB64, b64ToInt, codeB64ToB2, codeB2ToB64, Reb64, + nabSextets) + +from .. import kering +from ..kering import (Versionage, Vrsn_1_0, Vrsn_2_0) + +from ..core.coring import MapDom + + + +@dataclass(frozen=True) +class GenusCodex: + """GenusCodex is codex of protocol genera for code table. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + KERI_ACDC_SPAC: str = '--AAA' # KERI, ACDC, and SPAC Protocol Stacks share the same tables + KERI: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + ACDC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + SPAC: str = '--AAA' # KERI and ACDC Protocol Stacks share the same tables + + + def __iter__(self): + return iter(astuple(self)) # enables inclusion test with "in" + # duplicate values above just result in multiple entries in tuple so + # in inclusion still works + +GenDex = GenusCodex() # Make instance + + +@dataclass(frozen=True) +class CounterCodex_1_0(MapDom): + """ + CounterCodex is codex hard (stable) part of all counter derivation codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get codes with item syntax using tag variables. + Example: codex[tag] + """ + ControllerIdxSigs: str = '-A' # Qualified Base64 Indexed Signature. + WitnessIdxSigs: str = '-B' # Qualified Base64 Indexed Signature. + NonTransReceiptCouples: str = '-C' # Composed Base64 Couple, pre+cig. + TransReceiptQuadruples: str = '-D' # Composed Base64 Quadruple, pre+snu+dig+sig. + FirstSeenReplayCouples: str = '-E' # Composed Base64 Couple, fnu+dts. + TransIdxSigGroups: str = '-F' # Composed Base64 Group, pre+snu+dig+ControllerIdxSigs group. + SealSourceCouples: str = '-G' # Composed Base64 couple, snu+dig of given delegator/issuer/transaction event + TransLastIdxSigGroups: str = '-H' # Composed Base64 Group, pre+ControllerIdxSigs group. + SealSourceTriples: str = '-I' # Composed Base64 triple, pre+snu+dig of anchoring source event + SadPathSigGroups: str = '-J' # Composed Base64 Group path+TransIdxSigGroup of SAID of content + RootSadPathSigGroups: str = '-K' # Composed Base64 Group, root(path)+SaidPathCouples + PathedMaterialGroup: str = '-L' # Composed Grouped Pathed Material Quadlet (4 char each) + BigPathedMaterialGroup: str = '-0L' # Composed Grouped Pathed Material Quadlet (4 char each) + AttachmentGroup: str = '-V' # Composed Grouped Attached Material Quadlet (4 char each) + BigAttachmentGroup: str = '-0V' # Composed Grouped Attached Material Quadlet (4 char each) + ESSRPayloadGroup: str = '-Z' # ESSR Payload Group, dig of content+Texter group + KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Protocol Stack CESR Version + + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +CtrDex_1_0 = CounterCodex_1_0() + +@dataclass(frozen=True) +class CounterCodex_2_0(MapDom): + """ + CounterCodex is codex hard (stable) part of all counter derivation codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get codes with item syntax using tag variables. + Example: codex[tag] + """ + GenericGroup: str = '-A' # Generic Group (Universal with Override). + BigGenericGroup: str = '-0A' # Big Generic Group (Universal with Override). + MessageGroup: str = '-B' # Message Body plus Attachments Group (Universal with Override). + BigMessageGroup: str = '-0B' # Big Message Body plus Attachments Group (Universal with Override). + AttachmentGroup: str = '-C' # Message Attachments Only Group (Universal with Override). + BigAttachmentGroup: str = '-0C' # Big Attachments Only Group (Universal with Override). + DatagramSegmentGroup: str = '-D' # Datagram Segment Group (Universal). + BigDatagramSegmentGroup: str = '-0D' # Big Datagram Segment Group (Universal). + ESSRWrapperGroup: str = '-E' # ESSR Wrapper Group (Universal). + BigESSRWrapperGroup: str = '-0E' # Big ESSR Wrapper Group (Universal). + FixedMessageBodyGroup: str = '-F' # Fixed Field Message Body Group (Universal). + BigFixedMessageBodyGroup: str = '-0F' # Big Fixed Field Message Body Group (Universal). + MapMessageBodyGroup: str = '-G' # Field Map Message Body Group (Universal). + BigMapMessageBodyGroup: str = '-0G' # Big Field Map Message Body Group (Universal). + GenericMapGroup: str = '-H' # Generic Field Map Group (Universal). + BigGenericMapGroup: str = '-0H' # Big Generic Field Map Group (Universal). + GenericListGroup: str = '-I' # Generic List Group (Universal). + BigGenericListGroup: str = '-0I' # Big Generic List Group (Universal). + ControllerIdxSigs: str = '-J' # Controller Indexed Signature(s) of qb64. + BigControllerIdxSigs: str = '-0J' # Big Controller Indexed Signature(s) of qb64. + WitnessIdxSigs: str = '-K' # Witness Indexed Signature(s) of qb64. + BigWitnessIdxSigs: str = '-0K' # Big Witness Indexed Signature(s) of qb64. + NonTransReceiptCouples: str = '-L' # NonTrans Receipt Couple(s), pre+cig. + BigNonTransReceiptCouples: str = '-0L' # Big NonTrans Receipt Couple(s), pre+cig. + TransReceiptQuadruples: str = '-M' # Trans Receipt Quadruple(s), pre+snu+dig+sig. + BigTransReceiptQuadruples: str = '-0M' # Big Trans Receipt Quadruple(s), pre+snu+dig+sig. + FirstSeenReplayCouples: str = '-N' # First Seen Replay Couple(s), fnu+dts. + BigFirstSeenReplayCouples: str = '-0N' # First Seen Replay Couple(s), fnu+dts. + TransIdxSigGroups: str = '-O' # Trans Indexed Signature Group(s), pre+snu+dig+CtrControllerIdxSigs of qb64. + BigTransIdxSigGroups: str = '-0O' # Big Trans Indexed Signature Group(s), pre+snu+dig+CtrControllerIdxSigs of qb64. + TransLastIdxSigGroups: str = '-P' # Trans Last Est Evt Indexed Signature Group(s), pre+CtrControllerIdxSigs of qb64. + BigTransLastIdxSigGroups: str = '-0P' # Big Trans Last Est Evt Indexed Signature Group(s), pre+CtrControllerIdxSigs of qb64. + SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + BigSealSourceCouples: str = '-0Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + SealSourceTriples: str = '-R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + BigSealSourceTriples: str = '-0R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + PathedMaterialGroup: str = '-S' # Pathed Material Group. + BigPathedMaterialGroup: str = '-0S' # Big Pathed Material Group. + SadPathSigGroups: str = '-T' # SAD Path Group(s) sadpath+CtrTransIdxSigGroup(s) of SAID qb64 of content. + BigSadPathSigGroups: str = '-0T' # Big SAD Path Group(s) sadpath+CtrTransIdxSigGroup(s) of SAID qb64 of content. + RootSadPathSigGroups: str = '-U' # Root Path SAD Path Group(s), rootpath+SadPathGroup(s). + BigRootSadPathSigGroups: str = '-0U' # Big Root Path SAD Path Group(s), rootpath+SadPathGroup(s). + DigestSealSingles: str = '-V' # Digest Seal Single(s), dig of sealed data. + BigDigestSealSingles: str = '-0V' # Big Digest Seal Single(s), dig of sealed data. + MerkleRootSealSingles: str = '-W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BigMerkleRootSealSingles: str = '-0W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BackerRegistrarSealCouples: str = '-X' # Backer Registrar Seal Couple(s), brid+dig of sealed data. + BigBackerRegistrarSealCouples: str = '-0X' # Big Backer Registrar Seal Couple(s), brid+dig of sealed data. + SealSourceLastSingles: str = '-Y' # Seal Source Couple(s), pre of last source sealing or sealed event. + BigSealSourceLastSingles: str = '-0Y' # Big Seal Source Couple(s), pre of last source sealing or sealed event. + ESSRPayloadGroup: str = '-Z' # ESSR Payload Group. + BigESSRPayloadGroup: str = '-0Z' # Big ESSR Payload Group. + KERIACDCGenusVersion: str = '--AAA' # KERI ACDC Stack CESR Protocol Genus Version (Universal) + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +CtrDex_2_0 = CounterCodex_2_0() + +# CodeNames is tuple of codes names given by attributes of union of codices +CodeNames = tuple(asdict(CtrDex_2_0) | asdict(CtrDex_1_0)) +# Codens is namedtuple of CodeNames where its names are the code names +# Codens enables using the attributes of the named tuple to specify a code by +# name (indirection) so that changes in the code itself do not break the +# creation of a counter. Enables specifying a counter by the code name not the +# code itself. The code may change between versions but the code name does not. +Codenage = namedtuple("Codenage", CodeNames, defaults=CodeNames) +Codens = Codenage() + + +@dataclass(frozen=True) +class SealCodex_2_0(MapDom): + """ + SealCodex_2_0 is codex of seal counter derivation codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get codes with item syntax using tag variables. + Example: codex[tag] + """ + SealSourceCouples: str = '-Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + BigSealSourceCouples: str = '-0Q' # Seal Source Couple(s), snu+dig of source sealing or sealed event. + SealSourceTriples: str = '-R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + BigSealSourceTriples: str = '-0R' # Seal Source Triple(s), pre+snu+dig of source sealing or sealed event. + DigestSealSingles: str = '-V' # Digest Seal Single(s), dig of sealed data. + BigDigestSealSingles: str = '-0V' # Big Digest Seal Single(s), dig of sealed data. + MerkleRootSealSingles: str = '-W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BigMerkleRootSealSingles: str = '-0W' # Merkle Tree Root Digest Seal Single(s), dig of sealed data. + BackerRegistrarSealCouples: str = '-X' # Backer Registrar Seal Couple(s), brid+dig of sealed data. + BigBackerRegistrarSealCouples: str = '-0X' # Big Backer Registrar Seal Couple(s), brid+dig of sealed data. + SealSourceLastSingles: str = '-Y' # Seal Source Couple(s), pre of last source sealing event. + BigSealSourceLastSingles: str = '-0Y' # Big Seal Source Couple(s), pre of last source sealing event. + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +SealDex_2_0 = SealCodex_2_0() + +# namedtuple for size entries in Counter derivation code tables +# hs is the hard size int number of chars in hard (stable) part of code +# ss is the soft size int number of chars in soft (unstable) part of code +# fs is the full size int number of chars in code +Cizage = namedtuple("Cizage", "hs ss fs") + + +class Counter: + """ + Counter is fully qualified cryptographic material primitive base class for + counter primitives (framing composition grouping count codes). + + Sub classes are derivation code and key event element context specific. + + Includes the following attributes and properties: + + Class Attributes: + Codes (dict): nested of codexes keyed by major and minor version + Names (dict): nested of map of code names to codes keyed by + major and minor version + Hards (dict): of hard code sizes keyed by text domain selector + Bards (dict): of hard code sizes keyed by binary domain selector + Sizes (dict): of size tables keyed by version. Size table is dict + of Sizages keyed by hard code + + Attributes: + + + Properties: + .version (Versionage): current CESR code table protocol genus version + .codes (CounterCodex_1_0 | CounterCodex_1_0): version specific codex + .sizes (dict): version specific sizes table + .code (str): hard part of derivation code to indicate cypher suite + .raw (bytes): crypto material only without code + .pad (int): number of pad chars given raw + .count (int): count of quadlets/triplets of following framed material + (not including code) + .qb64 (str | bytes | bytearray): in Base64 fully qualified with + derivation code + crypto mat + .qb64b (bytes | bytearray): in Base64 fully qualified with + derivation code + crypto mat + .qb2 (bytes | bytearray): in binary with derivation code + + crypto material + + Hidden: + ._version (Versionage): value for .version property + ._codes (CounterCodex_1_0 | CounterCodex_1_0): version specific codex + ._sizes (dict): version specific sizes table + ._code (str): value for .code property + ._raw (bytes): value for .raw property + ._count (int): value for .count property + + + Versioning: + CESR Genus specific code tables have a major and a minor version. + + For a given major version all minor versions must be backwards compatible. + This means that minor version changes to tables are append only. New + codes may be added but no existing codes may be changed. This means that + a given implementation need only use use the latest minor version of + the code table for a given major version when generating or parsing a + primitive or group. Assuming the major versions match, when parsing, + a primitive, when that primitive was generated with a later minor version + than the implementation supports then it will not be recognized and + raise an error. But if a primitive was generated with any earlier minor + version than the version the implementation supports then the primitive + will parse correctly using any later minor version of the code table. + + Likewise a given protocol stack may have message bodies that carry + a major and a minor version. + + A given CESR Genus and a given Protocol message stack may be paired in + order to synchronize versioning between the two when the message bodies + use primitives and or groups defined by codes in the CESR Genus table. + + In this case pairing is between the CESR Genus labeled KERI_ACDC_SPAC + and the message body protocol stack labeled KERI/ACDC/SPAC + + The two versions, CESR Genus and Protocol Stack, may be synchronized in + the following way: + + * Major versions must match or be compatible + + * Minor versions may differ but must be compatible within a + major version. + + Importantly the CESR code table version may not be included in the + message body itself but only provided in the surrounding CESR stream. + This means the code table version used by a message body may not be + signed. Therefore the receiver of a message body with embedded CESR + primitives and groups must be protected from a CESR code table genus + version malleability attack. + + When the major versions of the CESR code table and protocol stack + match, the signed embedded protocol stack major version protects + the receiver from a major version malleability attack on the CESR + code table. Otherwise the major versions must be compatible in a way + that does not allow malleability. For example the set of allowed codes + for a given message protocol version are compatible across CESR code + table major versions. + + This, however, does not protect the receiver of a message body from + a minor version malleability attack on the CESR code table. + Nevertheless, the requirement that all minor versions of a CESR code + table for a given major version must be backwards compatible, + does indeed provide this protection. + + Either, the receiver of the message body recognizes exactly + all primitives and groups in the message body because the CESR code + table minor version supported by the receiver is greater than or equal + to that used by the the minor version of the sender or any unsupported + (later appended) primitives or group codes will be unrecognized by + the received thereby raising an error that results in the message being + dropped. + + """ + Codes = \ + { + Vrsn_1_0.major: \ + { + Vrsn_1_0.minor: CtrDex_1_0, + }, + Vrsn_2_0.major: \ + { + Vrsn_2_0.minor: CtrDex_2_0, + }, + } + + + # invert dataclass codenames: codes to dict codes: codenames + Names = copy.deepcopy(Codes) # make deep nested copy so can invert nested values + for minor in Names.values(): + for key in minor: + minor[key] = {val: key for key, val in asdict(minor[key]).items()} + + + + + # Hards table maps from bytes Base64 first two code chars to int of + # hard size, hs,(stable) of code. The soft size, ss, (unstable) for Counter + # is always > 0 and hs + ss = fs always + Hards = ({('-' + chr(c)): 2 for c in range(65, 65 + 26)}) + Hards.update({('-' + chr(c)): 2 for c in range(97, 97 + 26)}) + Hards.update([('-0', 3)]) + Hards.update([('--', 5)]) + + # Bards table maps to hard size, hs, of code from bytes holding sextets + # converted from first two code char. Used for ._bexfil. + Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + + # Sizes table indexes size tables first by major version and then by + # lastest minor version + # Each size table maps hs chars of code to Cizage namedtuple of (hs, ss, fs) + # where hs is hard size, ss is soft size, and fs is full size + # soft size, ss, should always be > 0 and hs+ss=fs for Counter + Sizes = \ + { + Vrsn_1_0.major: \ + { + Vrsn_1_0.minor: \ + { + '-A': Cizage(hs=2, ss=2, fs=4), + '-B': Cizage(hs=2, ss=2, fs=4), + '-C': Cizage(hs=2, ss=2, fs=4), + '-D': Cizage(hs=2, ss=2, fs=4), + '-E': Cizage(hs=2, ss=2, fs=4), + '-F': Cizage(hs=2, ss=2, fs=4), + '-G': Cizage(hs=2, ss=2, fs=4), + '-H': Cizage(hs=2, ss=2, fs=4), + '-I': Cizage(hs=2, ss=2, fs=4), + '-J': Cizage(hs=2, ss=2, fs=4), + '-K': Cizage(hs=2, ss=2, fs=4), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '--AAA': Cizage(hs=5, ss=3, fs=8), + }, + }, + Vrsn_2_0.major: \ + { + Vrsn_2_0.minor: \ + { + '-A': Cizage(hs=2, ss=2, fs=4), + '-0A': Cizage(hs=3, ss=5, fs=8), + '-B': Cizage(hs=2, ss=2, fs=4), + '-0B': Cizage(hs=3, ss=5, fs=8), + '-C': Cizage(hs=2, ss=2, fs=4), + '-0C': Cizage(hs=3, ss=5, fs=8), + '-D': Cizage(hs=2, ss=2, fs=4), + '-0D': Cizage(hs=3, ss=5, fs=8), + '-E': Cizage(hs=2, ss=2, fs=4), + '-0E': Cizage(hs=3, ss=5, fs=8), + '-F': Cizage(hs=2, ss=2, fs=4), + '-0F': Cizage(hs=3, ss=5, fs=8), + '-G': Cizage(hs=2, ss=2, fs=4), + '-0G': Cizage(hs=3, ss=5, fs=8,), + '-H': Cizage(hs=2, ss=2, fs=4), + '-0H': Cizage(hs=3, ss=5, fs=8), + '-I': Cizage(hs=2, ss=2, fs=4), + '-0I': Cizage(hs=3, ss=5, fs=8), + '-J': Cizage(hs=2, ss=2, fs=4,), + '-0J': Cizage(hs=3, ss=5, fs=8), + '-K': Cizage(hs=2, ss=2, fs=4), + '-0K': Cizage(hs=3, ss=5, fs=8), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-M': Cizage(hs=2, ss=2, fs=4), + '-0M': Cizage(hs=3, ss=5, fs=8), + '-N': Cizage(hs=2, ss=2, fs=4), + '-0N': Cizage(hs=3, ss=5, fs=8), + '-O': Cizage(hs=2, ss=2, fs=4), + '-0O': Cizage(hs=3, ss=5, fs=8), + '-P': Cizage(hs=2, ss=2, fs=4), + '-0P': Cizage(hs=3, ss=5, fs=8), + '-Q': Cizage(hs=2, ss=2, fs=4), + '-0Q': Cizage(hs=3, ss=5, fs=8), + '-R': Cizage(hs=2, ss=2, fs=4), + '-0R': Cizage(hs=3, ss=5, fs=8), + '-S': Cizage(hs=2, ss=2, fs=4), + '-0S': Cizage(hs=3, ss=5, fs=8), + '-T': Cizage(hs=2, ss=2, fs=4), + '-0T': Cizage(hs=3, ss=5, fs=8), + '-U': Cizage(hs=2, ss=2, fs=4), + '-0U': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-W': Cizage(hs=2, ss=2, fs=4), + '-0W': Cizage(hs=3, ss=5, fs=8), + '-X': Cizage(hs=2, ss=2, fs=4), + '-0X': Cizage(hs=3, ss=5, fs=8), + '-Y': Cizage(hs=2, ss=2, fs=4), + '-0Y': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '-0Z': Cizage(hs=3, ss=5, fs=8), + '--AAA': Cizage(hs=5, ss=3, fs=8), + }, + }, + } + + + def __init__(self, code=None, *, count=None, countB64=None, + qb64b=None, qb64=None, qb2=None, strip=False, + gvrsn=Vrsn_2_0, **kwa): + """ + Validate as fully qualified + Parameters: + code (str | None): either stable (hard) part of derivation code or + code name. When code name then look up code from + ._codes. This allows versioning to change code + but keep stable code name. + + count (int | None): count of framed material in quadlets/triplets + for composition. Count does not include code. + When both count and countB64 are None then count + defaults to 1 + countB64 (str | None): count of framed material in quadlets/triplets + for composition as Base64 representation of int. + qb64b (bytes | bytearray | None): fully qualified crypto material text domain + if code nor tag is provided + qb64 (str | None) fully qualified crypto material text domain + if code nor tag not qb64b is provided + qb2 (bytes | bytearray | None) fully qualified crypto material binary domain + if code nor tag not qb64b nor qb54 is provided + strip (bool): True means strip counter contents from input stream + bytearray after parsing qb64b or qb2. False means do not strip. + default False + gvrsn (Versionage): instance of genera version of CESR code tables + + + Needs either code or qb64b or qb64 or qb2 + Otherwise raises EmptyMaterialError + When code and count provided then validate that code and count are correct + Else when qb64b or qb64 or qb2 provided extract and assign + .code and .count + + """ + if gvrsn.major not in self.Sizes: + raise kering.InvalidVersionError(f"Unsupported major version=" + f"{gvrsn.major}.") + + latest = list(self.Sizes[gvrsn.major])[0] # get latest minor version + if gvrsn.minor > latest: + raise kering.InvalidVersionError(f"Minor version={gvrsn.minor} " + f" exceeds latest supported minor" + f" version={latest}.") + + self._codes = self.Codes[gvrsn.major][latest] # use latest supported version codes + self._sizes = self.Sizes[gvrsn.major][latest] # use latest supported version sizes + self._version = gvrsn # provided version may be earlier than supported version + + + if code: # code (hard) provided + # assumes ._sizes ._codes coherent + if code not in self._sizes or len(code) < 2: + try: + code = self._codes[code] # code is code name so look up code + if code not in self._sizes or len(code) < 2: + raise kering.InvalidCodeError(f"Unsupported {code=}.") + except Exception as ex: + raise kering.InvalidCodeError(f"Unsupported {code=}.") from ex + + hs, ss, fs = self._sizes[code] # get sizes for code + cs = hs + ss # both hard + soft code size + if hs < 2 or fs != cs or cs % 4: # fs must be bs and multiple of 4 for count codes + raise kering.InvalidCodeSizeError(f"Whole code size not full " + f"size or not multiple of 4. " + f"{cs=} {fs=}.") + + if count is None: + count = 1 if countB64 is None else b64ToInt(countB64) + + if code[1] not in ("123456789-_"): # small [A-Z,a-z] or large [0] + if ss not in (2, 5): # not valid dynamic soft sizes + raise kering.InvalidVarIndexError(f"Invalid {ss=} " + f"for {code=}.") + # dynamically promote code based on count + if code[1] != '0' and count > (64 ** 2 - 1): # small code but large count + # elevate code due to large count + code = f"-0{code[1]}" # promote hard + ss = 5 + + if count < 0 or count > (64 ** ss - 1): + raise kering.InvalidVarIndexError(f"Invalid {count=} for " + f"{code=} with {ss=}.") + + self._code = code + self._count = count + + elif qb64b is not None: + self._exfil(qb64b) + if strip: # assumes bytearray + del qb64b[:self._sizes[self.code].fs] + + elif qb64 is not None: + self._exfil(qb64) + + elif qb2 is not None: # rewrite to use direct binary exfiltration + self._bexfil(qb2) + if strip: # assumes bytearray + del qb2[:self._sizes[self.code].fs * 3 // 4] + + else: + raise kering.EmptyMaterialError("Improper initialization need either " + "(code and count) or qb64b or " + "qb64 or qb2.") + + self._name = self.Names[gvrsn.major][latest][self.code] + + @property + def version(self): + """ + Returns ._version + Makes .version read only + """ + return self._version + + @property + def gvrsn(self): + """ + Returns .version alias for .version + + """ + return self.version + + @property + def codes(self): + """ + Returns ._codes + Makes .codes read only + """ + return self._codes + + + @property + def sizes(self): + """ + Returns ._sizes + Makes .sizes read only + """ + return self._sizes + + @property + def code(self): + """ + Returns: + code (str): hard part only of full text code. + Getter for ._code. Makes .code read only + + Soft part is count + """ + return self._code + + @property + def name(self): + """ + Returns: + name (str): code name for self.code. Match interface + for annotation for primitives like Matter + + Getter for ._name. Makes .name read only + + """ + return self._name + + + @property + def hard(self): + """ + Returns: + hard (str): hard part only of full text code. Alias for .code. + + """ + return self.code + + + @property + def count(self): + """ + Returns: + count (int): count value in quadlets/triples chars/bytes of material + framed by counter. + Getter for ._count. Makes ._count read only + """ + return self._count + + + @property + def soft(self): + """ + Returns: + soft (str): Base64 soft part of full counter code. Count value in + quadlets/triples chars/bytes of material framed by counter. + Converts .count to b64 + """ + _, ss, _ = self.sizes[self.code] + return intToB64(self._count, l=ss) + + + @property + def both(self): + """ + Returns: + both (str): hard + soft parts of full text code + """ + return f"{self.hard}{self.soft}" + + + @property + def fullSize(self): + """ + Returns full size of counter in bytes + + """ + _, _, fs = self.sizes[self.code] # get from sizes table + + return fs + + + @property + def qb64b(self): + """ + Property qb64b: + Returns Fully Qualified Base64 Version encoded as bytes + Assumes self.raw and self.code are correctly populated + """ + return self._infil() + + + @property + def qb64(self): + """ + Property qb64: + Returns Fully Qualified Base64 Version + Assumes self.raw and self.code are correctly populated + """ + return self.qb64b.decode("utf-8") + + + @property + def qb2(self): + """ + Property qb2: + Returns Fully Qualified Binary Version Bytes + """ + return self._binfil() + + + def countToB64(self, l=None): + """ Returns count as Base64 left padded with "A"s + Parameters: + l (int | None): minimum number characters including left padding + When not provided use the softsize of .code + + """ + if l is None: + _, ss, _ = self._sizes[self.code] + l = ss + return (intToB64(self.count, l=l)) + + + @staticmethod + def verToB64(version=None, *, text="", major=0, minor=0): + """ Converts version to Base64 representation of countB64 + suitable for CESR protocol genus and version + + Returns: + countB64 (str): suitable for input to Counter + + Example: + Counter(countB64=Counter.verToB64(verstr = "1.0")) + + Parameters: + version (Versionage): instange of namedtuple + Versionage(major=major,minor=minor) + text (str): text format of version as dotted decimal "major.minor" + major (int): When version is None and verstr is empty then use major minor + range [0, 63] for one Base64 character + minor (int): When version is None and verstr is empty then use major minor + range [0, 4095] for two Base64 characters + + """ + if version: + major = version.major + minor = version.minor + + elif text: + splits = text.split(".", maxsplit=2) + splits = [(int(s) if s else 0) for s in splits] + parts = [major, minor] + for i in range(2-len(splits),0, -1): # append missing minor and/or major + splits.append(parts[-i]) + major = splits[0] + minor = splits[1] + + if major < 0 or major > 63 or minor < 0 or minor > 4095: + raise ValueError(f"Out of bounds version = {major}.{minor}.") + + return (f"{intToB64(major)}{intToB64(minor, l=2)}") + + + @staticmethod + def b64ToVer(b64, *, texted=False): + """ Converts Base64 representation of version to Versionage or + text dotted decimal format + + default is Versionage + + Returns: + version (Versionage | str): + + Example: + Counter(version=Counter.b64ToVer("BAA")) + + Parameters: + b64 (str): base64 string of three characters Mmm for Major minor + texted (bool): return text format dotted decimal string + + + """ + if not Reb64.match(b64.encode("utf-8")): + raise ValueError("Invalid Base64.") + + if texted: + return ".".join([f"{b64ToInt(b64[0])}", f"{b64ToInt(b64[1:3])}"]) + + return Versionage(major=b64ToInt(b64[0]), minor=b64ToInt(b64[1:3])) + + + def _infil(self): + """ + Returns fully qualified attached sig base64 bytes computed from + self.code and self.count. + """ + code = self.code # codex value chars hard code + count = self.count # index value int used for soft + + hs, ss, fs = self._sizes[code] + # assumes fs = hs + ss # both hard + soft size + # assumes unit tests ensure ._sizes table entries are consistent + # hs >= 2, ss > 0 fs == hs + ss, not (fs % 4) + + if count < 0 or count > (64 ** ss - 1): + raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + + # both is hard code + converted count + both = "{}{}".format(code, intToB64(count, l=ss)) + + # check valid pad size for whole code size + if len(both) % 4: # no pad + raise kering.InvalidCodeSizeError("Invalid size = {} of {} not a multiple of 4." + .format(len(both), both)) + # prepending full derivation code with index and strip off trailing pad characters + return (both.encode("utf-8")) + + + def _binfil(self): + """ + Returns bytes of fully qualified base2 bytes, that is .qb2 + self.code converted to Base2 left shifted with pad bits + equivalent of Base64 decode of .qb64 into .qb2 + """ + code = self.code # codex chars hard code + count = self.count # index value int used for soft + + hs, ss, fs = self._sizes[code] + # assumes fs = hs + ss + # assumes unit tests ensure ._sizes table entries are consistent + # hs >= 2, ss>0 fs == hs + ss, not (fs % 4) + + if count < 0 or count > (64 ** ss - 1): + raise kering.InvalidVarIndexError("Invalid count={} for code={}.".format(count, code)) + + # both is hard code + converted count + both = "{}{}".format(code, intToB64(count, l=ss)) + if len(both) != fs: + raise kering.InvalidCodeSizeError("Mismatch code size = {} with table = {}." + .format(fs, len(both))) + + return (codeB64ToB2(both)) # convert to b2 left shift if any + + + def _exfil(self, qb64b): + """ + Extracts self.code and self.count from qualified base64 bytes qb64b + """ + if not qb64b: # empty need more bytes + raise kering.ShortageError("Empty material, Need more characters.") + + first = qb64b[:2] # extract first two char code selector + if hasattr(first, "decode"): + first = first.decode("utf-8") + if first not in self.Hards: + if first[0] == '_': + raise kering.UnexpectedOpCodeError("Unexpected op code start" + "while extracing Counter.") + else: + raise kering.UnexpectedCodeError("Unsupported code start ={}.".format(first)) + + hs = self.Hards[first] # get hard code size + if len(qb64b) < hs: # need more bytes + raise kering.ShortageError("Need {} more characters.".format(hs - len(qb64b))) + + hard = qb64b[:hs] # get hard code + if hasattr(hard, "decode"): + hard = hard.decode("utf-8") # decode converts bytearray/bytes to str + if hard not in self._sizes: # Sizes needs str not bytes + raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) + + hs, ss, fs = self._sizes[hard] # assumes hs consistent in both tables + # assumes fs = hs + ss # both hard + soft code size + # assumes that unit tests on Counter and CounterCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 + + if len(qb64b) < fs: # need more bytes + raise kering.ShortageError("Need {} more characters.".format(fs - len(qb64b))) + + count = qb64b[hs:fs] # extract count chars + if hasattr(count, "decode"): + count = count.decode("utf-8") + count = b64ToInt(count) # compute int count + + self._code = hard + self._count = count + + + def _bexfil(self, qb2): + """ + Extracts self.code and self.count from qualified base2 bytes qb2 + """ + if not qb2: # empty need more bytes + raise kering.ShortageError("Empty material, Need more bytes.") + + first = nabSextets(qb2, 2) # extract first two sextets as code selector + if first not in self.Bards: + if first[0] == b'\xfc': # b64ToB2('_') + raise kering.UnexpectedOpCodeError("Unexpected op code start" + "while extracing Matter.") + else: + raise kering.UnexpectedCodeError("Unsupported code start sextet={}.".format(first)) + + hs = self.Bards[first] # get code hard size equvalent sextets + bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets + if len(qb2) < bhs: # need more bytes + raise kering.ShortageError("Need {} more bytes.".format(bhs - len(qb2))) + + hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code + if hard not in self._sizes: + raise kering.UnexpectedCodeError("Unsupported code ={}.".format(hard)) + + hs, ss, fs = self._sizes[hard] + # assumes fs = hs + ss # both hs and ss + # assumes that unit tests on Counter and CounterCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 + + bcs = sceil(fs * 3 / 4) # bcs is min bytes to hold fs sextets + if len(qb2) < bcs: # need more bytes + raise kering.ShortageError("Need {} more bytes.".format(bcs - len(qb2))) + + both = codeB2ToB64(qb2, fs) # extract and convert both hard and soft part of code + count = b64ToInt(both[hs:fs]) # get count + + self._code = hard + self._count = count + diff --git a/src/keri/core/eventing.py b/src/keri/core/eventing.py index 5bbe5f2ca..6628352c1 100644 --- a/src/keri/core/eventing.py +++ b/src/keri/core/eventing.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- """ keri.core.eventing module @@ -7,59 +7,50 @@ import json import logging from collections import namedtuple -from dataclasses import dataclass, astuple +from dataclasses import asdict from urllib.parse import urlsplit from math import ceil from ordered_set import OrderedSet as oset from hio.help import decking -from . import coring -from .coring import (versify, Serials, Ilks, PreDex, DigDex, - NonTransDex, CtrDex, Counter, - Number, Seqner, Siger, Cigar, Dater, - Verfer, Diger, Prefixer, Tholder, Saider) -from . import serdering -from .. import help -from .. import kering -from ..db import basing, dbing -from ..db.basing import KeyStateRecord, StateEERecord -from ..db.dbing import dgKey, snKey, fnKey, splitKeySN, splitKey +from .. import kering, core from ..kering import (MissingEntryError, ValidationError, MissingSignatureError, MissingWitnessSignatureError, UnverifiedReplyError, MissingDelegationError, OutOfOrderError, LikelyDuplicitousError, UnverifiedWitnessReceiptError, - UnverifiedReceiptError, UnverifiedTransferableReceiptError, QueryNotFoundError) -from ..kering import Version, Versionage -from ..kering import (ICP_LABELS, DIP_LABELS, IXN_LABELS) + UnverifiedReceiptError, UnverifiedTransferableReceiptError, + QueryNotFoundError, MisfitEventSourceError, + MissingDelegableApprovalError) +from ..kering import Version, Versionage, TraitDex +from .. import help from ..help import helping -logger = help.ogler.getLogger() +from . import coring +from .coring import (versify, Kinds, Ilks, PreDex, DigDex, + NonTransDex, + Number, Seqner, Cigar, Dater, + Verfer, Diger, Prefixer, Tholder, Saider) -EscrowTimeoutPS = 3600 # seconds for partial signed escrow timeout +from .counting import Counter, Codens -MaxIntThold = 2 ** 32 - 1 +from . import indexing +from .indexing import Siger -@dataclass(frozen=True) -class TraitCodex: - """ - TraitCodex is codex of inception configuration trait code strings - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. +from . import serdering + +from ..db import basing, dbing, subing +from ..db.basing import KeyStateRecord, StateEERecord, OobiRecord +from ..db.dbing import dgKey, snKey, fnKey, splitSnKey, splitKey - """ - EstOnly: str = 'EO' # Only allow establishment events - DoNotDelegate: str = 'DND' # Dot not allow delegated identifiers - NoBackers: str = 'NB' # Do not allow any registrar backers - Backers: str = 'RB' # Registrar backer provided in Registrar seal - def __iter__(self): - return iter(astuple(self)) +logger = help.ogler.getLogger() +EscrowTimeoutPS = 3600 # seconds for partial signed escrow timeout -TraitDex = TraitCodex() # Make instance +MaxIntThold = 2 ** 32 - 1 # Location of last establishment key event: sn is int, dig is qb64 digest LastEstLoc = namedtuple("LastEstLoc", 's d') @@ -71,7 +62,7 @@ def __iter__(self): # sealdict =seal._asdict() # to convet dict to namedtuple use ** unpacking as in seal = SealDigest(**sealdict) # to check if dict of seal matches fields of associted namedtuple -# if tuple(sealdict.keys()) == SealEvent._fields: +# if tuple(sealdict) == SealEvent._fields: # Digest Seal: uniple (d,) # d = digest qb64 of data (usually SAID) @@ -86,85 +77,41 @@ def __iter__(self): # d = digest qb64 of backer metadata anchored to event usually SAID of data SealBacker = namedtuple("SealBacker", 'bi d') -# Event Seal: triple (i, s, d) -# i = pre is qb64 of identifier prefix of KEL for event, -# s = sn of event as lowercase hex string no leading zeros, -# d = SAID digest qb64 of event -SealEvent = namedtuple("SealEvent", 'i s d') - # Last Estalishment Event Seal: uniple (i,) # i = pre is qb64 of identifier prefix of KEL from which to get last est, event # used to indicate to get the latest keys available from KEL for 'i' SealLast = namedtuple("SealLast", 'i') -# Establishment Event for Source of Message: duple (s, d) +# Transaction Event Seal for Transaction Event: duple (s, d) +# s = sn of transaction event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of transaction event +# the pre is provided in the 'i' field qb64 of identifier prefix of KEL +# key event that this seal appears. +# use SealSourceCouples count code for attachment +SealTrans = namedtuple("SealTrans", 's d') + +# Event Seal: triple (i, s, d) +# i = pre is qb64 of identifier prefix of KEL for event, # s = sn of event as lowercase hex string no leading zeros, # d = SAID digest qb64 of event -# the pre is provided in the 'i' field of the message itself which is the qb64 -# of identifier prefix of KEL from which to get est, event given by 's d' -# use SealSourceCouples count code for attachment -SealEst = namedtuple("SealEst", 's d') +SealEvent = namedtuple("SealEvent", 'i s d') -# State (latest current) Event: triple (s, t, d) -# s = sn of latest event as lowercase hex string no leading zeros, -# t = message type of latest event (ilk) -# d = SAID digest qb64 of latest event -StateEvent = namedtuple("StateEvent", 's t d') +# Following are not seals only used in database -# State (latest current) Establishment Event: quadruple (s, d, br, ba) +# State Establishment Event (latest current) : quadruple (s, d, br, ba) # s = sn of latest est event as lowercase hex string no leading zeros, # d = SAID digest qb64 of latest establishment event # br = backer (witness) remove list (cuts) from latest est event # ba = backer (witness) add list (adds) from latest est event StateEstEvent = namedtuple("StateEstEvent", 's d br ba') +# not used should this be depricated? +# State Event (latest current) : triple (s, t, d) +# s = sn of latest event as lowercase hex string no leading zeros, +# t = message type of latest event (ilk) +# d = SAID digest qb64 of latest event +StateEvent = namedtuple("StateEvent", 's t d') -@dataclass(frozen=True) -class ColdCodex: - """ - ColdCodex is codex of cold stream start tritets of first byte - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - - First three bits: - 0o0 = 000 free - 0o1 = 001 cntcode B64 - 0o2 = 010 opcode B64 - 0o3 = 011 json - 0o4 = 100 mgpk - 0o5 = 101 cbor - 0o6 = 110 mgpk - 007 = 111 cntcode or opcode B2 - - status is one of ('evt', 'txt', 'bny' ) - 'evt' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) - 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) - 'bny' if tritet in (ColdDex.CtOpB2,) - - otherwise raise ColdStartError - - x = bytearray([0x2d, 0x5f]) - x == bytearray(b'-_') - x[0] >> 5 == 0o1 - True - """ - Free: int = 0o0 # not taken - CtB64: int = 0o1 # CountCode Base64 - OpB64: int = 0o2 # OpCode Base64 - JSON: int = 0o3 # JSON Map Event Start - MGPK1: int = 0o4 # MGPK Fixed Map Event Start - CBOR: int = 0o5 # CBOR Map Event Start - MGPK2: int = 0o6 # MGPK Big 16 or 32 Map Event Start - CtOpB2: int = 0o7 # CountCode or OpCode Base2 - - def __iter__(self): - return iter(astuple(self)) - - -ColdDex = ColdCodex() # Make instance - -Coldage = namedtuple("Coldage", 'msg txt bny') # stream cold start status -Colds = Coldage(msg='msg', txt='txt', bny='bny') # Future make Cues dataclasses instead of dicts. Dataclasses so may be converted @@ -413,44 +360,6 @@ def deTransReceiptQuintuple(data, strip=False): return esaider, sprefixer, sseqner, ssaider, siger -def validateSN(sn, inceptive=None): - """ - Returns: - sn (int): converted from sn hex str - - Raises ValueError if invalid sn - - Parameters: - sn (str): hex char sequence number of event or seal in an event - inceptive(bool): Check sn value and raise ValueError if invalid - None means check for sn < 0 - True means check for sn != 0 - False means check for sn < 1 - - """ - if len(sn) > 32: - raise ValueError("Invalid sn = {} too large.".format(sn)) - - try: - sn = int(sn, 16) - except Exception as ex: - raise ValueError("Invalid sn = {}.".format(sn)) - - if inceptive is not None: - if inceptive: - if sn != 0: - raise ValidationError("Nonzero sn = {} for inception evt." - "".format(sn)) - else: - if sn < 1: - raise ValidationError("Zero or less sn = {} for non-inception evt." - "".format(sn)) - else: - if sn < 0: - raise ValidationError("Negative sn = {} for event.".format(sn)) - - return sn - def verifySigs(raw, sigers, verfers): """ @@ -482,7 +391,7 @@ def verifySigs(raw, sigers, verfers): # verfer to each siger for siger in usigers: if siger.index >= len(verfers): - logger.info("Skipped sig: Index=%s to large.\n", siger.index) + logger.info("Skipped sig: Index=%s to large.", siger.index) siger.verfer = verfers[siger.index] # assign verfer # create lists of unique verified signatures and indices @@ -597,7 +506,7 @@ def state(pre, cnfg=None, # default to [] dpre=None, version=Version, - kind=Serials.json, + kind=Kinds.json, intive = False, ): """ @@ -735,7 +644,8 @@ def state(pre, ) return ksr # return KeyStateRecord use asdict(ksr) to get dict version - +# should remove intive as its not standard KERI so confusing and leads to errors +# this is an old feature that is now deprecated. def incept(keys, *, @@ -747,7 +657,7 @@ def incept(keys, cnfg=None, data=None, version=Version, - kind=Serials.json, + kind=Kinds.json, code=None, intive=False, delpre=None, @@ -758,9 +668,9 @@ def incept(keys, Parameters: keys (list): current signing keys qb64 - sith (int | str | list | None): current signing threshold input to Tholder + isith (int | str | list | None): current signing threshold input to Tholder ndigs (list | None): current signing key digests qb64 - nsith int | str | list | None): next signing threshold input to Tholder + nsith (int | str | list | None): next signing threshold input to Tholder toad (int | str | None): witness threshold number if str then hex str wits (list | None): witness identifier prefixes qb64 cnfg (list | None): configuration traits from TraitDex @@ -851,35 +761,8 @@ def incept(keys, saids = {'i': code} serder = serdering.SerderKERI(sad=ked, makify=True, saids=saids) - serder._verify() # raises error if fails verifications return serder - #if delpre is not None: # delegated inception with ilk = dip - #ked['di'] = delpre - #if code is None: - #code = MtrDex.Blake3_256 # force digestive - - #if delpre is None and code is None and len(keys) == 1: - #prefixer = Prefixer(qb64=keys[0]) # defaults to not digestive code - #if prefixer.digestive: - #raise ValueError("Invalid code, digestive={}, must be derived from" - #" ked.".format(prefixer.code)) - #else: # digestive - ## raises derivation error if non-empty nxt but ephemeral code - #prefixer = Prefixer(ked=ked, code=code) # Derive AID from ked and code - - #if delpre is not None: - #if not prefixer.digestive: - #raise ValueError(f"Invalid derivation code = {prefixer.code} " - #f"for delegation. Must be digestive") - - #ked["i"] = prefixer.qb64 # update pre element in ked with pre qb64 - #if prefixer.digestive: - #ked["d"] = prefixer.qb64 - #else: - #_, ked = coring.Saider.saidify(sad=ked) - - #return Serder(ked=ked) # return serialized ked def delcept(keys, delpre, **kwa): """ @@ -889,7 +772,7 @@ def delcept(keys, delpre, **kwa): Parameters: keys (list): current signing keys qb64 - sith (int | str | list | None): current signing threshold input to Tholder + isith (int | str | list | None): current signing threshold input to Tholder ndigs (list | None): current signing key digests qb64 nsith int | str | list | None): next signing threshold input to Tholder toad (int | str | None): witness threshold number if str then hex str @@ -922,7 +805,7 @@ def rotate(pre, adds=None, data=None, version=Version, - kind=Serials.json, + kind=Kinds.json, intive = False, ): """ @@ -935,9 +818,9 @@ def rotate(pre, dig (str): SAID of previous event qb64 ilk (str): ilk of event. Must be in (Ilks.rot, Ilks.drt) sn (int | str): sequence number int or hex str - sith (int | str | list | None): current signing threshold input to Tholder + isith (int | str | list | None): current signing threshold input to Tholder ndigs (list | None): current signing key digests qb64 - nsith int | str | list | None): next signing threshold input to Tholder + nsith (int | str | list | None): next signing threshold input to Tholder toad (int | str | None): witness threshold number if str then hex str wits (list | None): prior witness identifier prefixes qb64 cuts (list | None): witness prefixes to cut qb64 @@ -1042,12 +925,8 @@ def rotate(pre, ) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder - #_, ked = coring.Saider.saidify(sad=ked) - - #return Serder(ked=ked) # return serialized ked def deltate(pre, keys, @@ -1067,7 +946,7 @@ def deltate(pre, dig (str): said of previous event qb64 ilk (str): ilk of event. Must be in (Ilks.rot, Ilks.drt) sn (int | str): sequence number int or hex str - sith (int | str | list): current signing threshold input to Tholder + isith (int | str | list): current signing threshold input to Tholder ndigs (list): current signing key digests qb64 nsith int | str | list): next signing threshold input to Tholder toad (int | str ): witness threshold number if str then hex str @@ -1090,7 +969,7 @@ def interact(pre, sn=1, data=None, version=Version, - kind=Serials.json, + kind=Kinds.json, ): """ Returns serder of interaction event message. @@ -1123,20 +1002,15 @@ def interact(pre, ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder - #_, ked = coring.Saider.saidify(sad=ked) - - #return Serder(ked=ked) # return serialized ked - def receipt(pre, sn, said, *, version=Version, - kind=Serials.json + kind=Kinds.json ): """ Returns serder of event receipt message. Used for both non-trans and trans @@ -1166,7 +1040,6 @@ def receipt(pre, ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -1175,7 +1048,7 @@ def query(route="", query=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of query 'qry' message. Utility function to automate creation of query messages. @@ -1221,7 +1094,6 @@ def query(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder #_, ked = coring.Saider.saidify(sad=ked) @@ -1233,7 +1105,7 @@ def reply(route="", data=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of reply 'rpy' message. Utility function to automate creation of reply messages. @@ -1278,25 +1150,15 @@ def reply(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder - #_, sad = coring.Saider.saidify(sad=sad, kind=kind, label=label) - - #saider = coring.Saider(qb64=sad[label]) - #if not saider.verify(sad=sad, kind=kind, label=label, prefixed=True): - #raise ValidationError("Invalid said = {} for reply msg={}." - #"".format(saider.qb64, sad)) - - #return Serder(ked=sad) # return serialized Self-Addressed Data (SAD) - def prod(route="", replyRoute="", query=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of prod, 'pro', msg to request disclosure via bare, 'bar' msg of data anchored via seal(s) on KEL for identifier prefix, pre, when given @@ -1329,7 +1191,6 @@ def prod(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder #_, ked = coring.Saider.saidify(sad=ked) @@ -1340,7 +1201,7 @@ def bare(route="", data=None, stamp=None, version=Version, - kind=Serials.json): + kind=Kinds.json): """ Returns serder of bare 'bar' message. Utility function to automate creation of unhiding (bareing) messages for @@ -1389,7 +1250,6 @@ def bare(route="", ) serder = serdering.SerderKERI(sad=sad, makify=True) - serder._verify() # raises error if fails verifications return serder #_, sad = coring.Saider.saidify(sad=sad) @@ -1431,21 +1291,25 @@ def messagize(serder, *, sigers=None, seal=None, wigers=None, cigars=None, if sigers: if isinstance(seal, SealEvent): - atc.extend(Counter(CtrDex.TransIdxSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(seal.i.encode("utf-8")) atc.extend(Seqner(snh=seal.s).qb64b) atc.extend(seal.d.encode("utf-8")) elif isinstance(seal, SealLast): - atc.extend(Counter(CtrDex.TransLastIdxSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.TransLastIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(seal.i.encode("utf-8")) - atc.extend(Counter(code=CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(Counter(Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) if wigers: - atc.extend(Counter(code=CtrDex.WitnessIdxSigs, count=len(wigers)).qb64b) + atc.extend(Counter(Codens.WitnessIdxSigs, count=len(wigers), + gvrsn=kering.Vrsn_1_0).qb64b) for wiger in wigers: if wiger.verfer and wiger.verfer.code not in NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -1453,7 +1317,8 @@ def messagize(serder, *, sigers=None, seal=None, wigers=None, cigars=None, atc.extend(wiger.qb64b) if cigars: - atc.extend(Counter(code=CtrDex.NonTransReceiptCouples, count=len(cigars)).qb64b) + atc.extend(Counter(Codens.NonTransReceiptCouples, count=len(cigars), + gvrsn=kering.Vrsn_1_0).qb64b) for cigar in cigars: if cigar.verfer.code not in NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -1465,8 +1330,8 @@ def messagize(serder, *, sigers=None, seal=None, wigers=None, cigars=None, if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(Counter(code=CtrDex.AttachedMaterialQuadlets, - count=(len(atc) // 4)).qb64b) + msg.extend(Counter(Codens.AttachmentGroup, + count=(len(atc) // 4), gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(atc) return msg @@ -1497,33 +1362,40 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): count = 0 for (pather, sigers) in sadsigers: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSig, count=1).qb64b) + atc.extend(Counter(Codens.SadPathSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(pather.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(Counter(Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) for (pather, prefixer, seqner, saider, sigers) in sadtsgs: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSig, count=1).qb64b) + atc.extend(Counter(Codens.SadPathSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(pather.qb64b) - atc.extend(coring.Counter(coring.CtrDex.TransIdxSigGroups, count=1).qb64b) + atc.extend(Counter(Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(Counter(Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) for (pather, cigars) in sadcigars: count += 1 - atc.extend(coring.Counter(coring.CtrDex.SadPathSig, count=1).qb64b) + atc.extend(Counter(Codens.SadPathSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(pather.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=len(sadcigars)).qb64b) + atc.extend(Counter(Codens.NonTransReceiptCouples, + count=len(sadcigars), gvrsn=kering.Vrsn_1_0).qb64b) for cigar in cigars: if cigar.verfer.code not in coring.NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -1537,12 +1409,13 @@ def proofize(sadtsgs=None, *, sadsigers=None, sadcigars=None, pipelined=False): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, - count=(len(atc) // 4)).qb64b) + msg.extend(Counter(Codens.AttachmentGroup, count=(len(atc) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) if count > 1: root = coring.Pather(bext="-") - msg.extend(coring.Counter(code=coring.CtrDex.SadPathSigGroup, count=count).qb64b) + msg.extend(Counter(Codens.RootSadPathSigGroups, count=count, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(root.qb64b) msg.extend(atc) @@ -1572,10 +1445,6 @@ class Kever: qb64 identifier prefixes of own habitat identifiers. Assign db.prefixes when None When empty operate in promiscuous mode - local (bool): Injected from kevery when provided. - True means only process msgs for own events when .prefixes is not empty - False means only process msgs for not own events when .prefixes is not empty - Default is False. version (Versionage): serder.version instance of current event state version prefixer (Prefixer): instance for current event state sner (Number): instance of sequence number @@ -1601,7 +1470,7 @@ class Kever: lastEst (LastEstLoc): namedtuple of int sn .s and qb64 digest .d of last est event delegated (bool): True means delegated identifier, False not delegated - delgator (str): qb64 of delegator's prefix + delpre(str): qb64 of delegator's prefix Properties: @@ -1625,7 +1494,7 @@ class Kever: def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, db=None, estOnly=None, delseqner=None, delsaider=None, firner=None, - dater=None, cues=None, prefixes=None, local=False, check=False): + dater=None, cues=None, eager=False, local=True, check=False): """ Create incepting kever and state from inception serder Verify incepting serder against sigers raises ValidationError if not @@ -1653,15 +1522,15 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, When dater provided then use dater for first seen datetime cues (Deck | None): reference to Kevery.cues Deck when provided i.e. notices of events or requests to respond to - prefixes (list | None): own prefixes for own local habitats. - May not include the prefix of this Kever's event when inception - has not yet been accepted into KEL - Some restrictions if present - If empty then effectively in promiscuous mode - local (bool): True means only process msgs for own controller's - events if .prefixes is not empty. - False means only process msgs for not own events - if .prefixes is not empty + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote check (bool): True means do not update the database in any non-idempotent way. Useful for reinitializing the Kevers from a persisted KEL without updating non-idempotent first seen .fels @@ -1675,8 +1544,7 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, db = basing.Baser(reopen=True) # default name = "main" self.db = db self.cues = cues - self.prefixes = prefixes if prefixes is not None else db.prefixes - self.local = True if local else False + local = True if local else False if state: # preload from state self.reload(state) @@ -1692,19 +1560,14 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, ilk, serder.ked)) self.ilk = ilk - labels = DIP_LABELS if ilk == Ilks.dip else ICP_LABELS - for k in labels: - if k not in serder.ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, serder.ked)) - self.incept(serder=serder) # do major event validation and state setting self.config(serder=serder, estOnly=estOnly) # assign config traits perms # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, delegator, wigers = self.valSigsDelWigs(serder=serder, + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, sigers=sigers, verfers=serder.verfers, tholder=self.tholder, @@ -1712,19 +1575,22 @@ def __init__(self, *, state=None, serder=None, sigers=None, wigers=None, toader=self.toader, wits=self.wits, delseqner=delseqner, - delsaider=delsaider) + delsaider=delsaider, + eager=eager, + local=local) - self.delegator = delegator - self.delegated = True if self.delegator else False + self.delpre = delpre # may be None + self.delegated = True if self.delpre else False wits = serder.backs # serder.ked["b"] # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen # returns fn == None if already logged fn log is non idempotent - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, first=True if not check else False, seqner=delseqner, saider=delsaider, - firner=firner, dater=dater) + firner=firner, dater=dater, local=local) if fn is not None: # first is non-idempotent for fn check mode fn is None self.fner = Number(num=fn) self.dater = Dater(dts=dts) @@ -1767,6 +1633,22 @@ def kevers(self): return self.db.kevers + @property + def prefixes(self): + """ + Returns .db.prefixes + """ + return self.db.prefixes + + + @property + def groups(self): + """ + Returns .db.gids oset of group hab ids (prefixes) + """ + return self.db.groups + + @property def transferable(self): """ @@ -1778,33 +1660,107 @@ def transferable(self): return True if self.ndigers and self.prefixer.transferable else False - def locallyOwned(self, pre=''): - """Returns True if pre is in .prefixes False otherwise. Indicates that - provided identifier prefix is controlled by a local controller from - .prefixes + def locallyOwned(self, pre: str | None = None): + """Returns True if pre is in .prefixes and not in .groups + False otherwise. + Indicates that provided identifier prefix is controlled by a local + controller from .prefixes but is not a group with local member. i.e pre is a locally owned (controlled) AID (identifier prefix) + Returns: + (bool): True if pre is local hab but not group hab + When pre="" empty then returns False + Parameters: - pre (str): qb64 identifier prefix + pre (str|None): qb64 identifier prefix if any. Default None + None means use self.prefixer.qb64 + + + """ + pre = pre if pre is not None else self.prefixer.qb64 + return pre in self.prefixes and pre not in self.groups + + + def locallyDelegated(self, pre: str): + """Returns True if pre is in .prefixes and not in .groups + False otherwise. Use when pre is a delegator for some event and + want to confirm that pre is also locallyOwned thereby making the + associated event locallyDelegated. + + Indicates that provided identifier prefix is controlled by a local + controller from .prefixes but is not a group with local member. + i.e pre is a locally owned (controlled) AID (identifier prefix) + Because delpre may be None, changes the default to "" instead of + self.prefixer.pre because self.prefixer.pre is delegate not delegator + of self. Unaccepted dip events do not have self.delpre set yet. + + Returns: + (bool): True if pre is local hab but not group hab + When pre="" empty or None then returns False + Parameters: + pre (str): qb64 identifier prefix if any. """ return pre in self.prefixes - def locallyWitnessed(self, serder=None): + def locallyWitnessed(self, *, wits: list[str]=None, serder: (str)=None): """Returns True if a local controller is a witness of this Kever's KEL of wits in serder of if None then current wits for this Kever. i.e. self is witnessd by locally owned (controlled) AID (identifier prefix) Parameters: + wits (list[str]): qb64 identifier prefixes of witnesses serder ( SerderKERI | None): SerderKERI instace if any """ - if serder and serder.pre != self.prefixer.qb64: # same KEL as self - return False - wits = serder.backs if serder is not None else self.wits - return (oset(self.prefixes) & oset(wits)) + if not wits: + if not serder: + wits = self.wits + else: + if serder.pre != self.prefixer.qb64: # not same KEL as self + return False + wits, _, _ = self.deriveBacks(serder=serder) + + return True if (self.prefixes & oset(wits)) else False + + + def locallyMembered(self, pre: str | None = None): + """Returns True if group hab identifier prefix pre has as a contributing + member a locally owned prefix by virture of pre in .groups + + Returns: + (bool): True if pre is group hab identifier in .groups + False otherwise + + Parameters: + pre (str|None): qb64 identifier prefix if any or None + When None default to use self.prefixer.qb64 + + """ + # assumes stale group membership is taken care of by presence of groups + # i.e where once a local member but no more. + pre = pre if pre is not None else self.prefixer.qb64 + return pre in self.groups # groups + + def locallyContributedIndices(self, verfers: list[Verfer]): + """Returns list of indices of public keys contributed by local members + to the KEL with current signing keys represented by verfers + + Using the pubs index to find members of a signing group + + Parameters: + verfers (list[Verfer]): instance for each current signing key + + Returns: + indices list[int]: list of indices of keys contributed by local members + + """ + habord = self.db.habs.get(keys=(self.prefixer.qb64,)) + kever = self.kevers[habord.mid] + idx = [verfer.qb64 for verfer in verfers].index(kever.verfers[0].qb64) + return [idx] def reload(self, state): """ @@ -1829,18 +1785,19 @@ def reload(self, state): self.cuts = state.ee.br self.adds = state.ee.ba self.estOnly = False - self.doNotDelegate = True if TraitCodex.DoNotDelegate in state.c else False - self.estOnly = True if TraitCodex.EstOnly in state.c else False + self.doNotDelegate = True if TraitDex.DoNotDelegate in state.c else False + self.estOnly = True if TraitDex.EstOnly in state.c else False self.lastEst = LastEstLoc(s=int(state.ee.s, 16), d=state.ee.d) - self.delegator = state.di if state.di else None - self.delegated = True if self.delegator else False + self.delpre = state.di if state.di else None + self.delegated = True if self.delpre else False if (raw := self.db.getEvt(key=dgKey(pre=self.prefixer.qb64, dig=state.d))) is None: raise MissingEntryError(f"Corresponding event not found for state=" f"{state}.") self.serder = serdering.SerderKERI(raw=bytes(raw)) + # May want to do additional checks here @@ -1870,10 +1827,6 @@ def incept(self, serder, estOnly=None): self.prefixer = Prefixer(qb64=serder.pre) - if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix - raise ValidationError("Invalid prefix = {} for inception evt = {}." - "".format(self.prefixer.qb64, ked)) - self.serder = serder # need whole serder for digest agility comparisons ndigs = serder.ndigs # ked["n"] @@ -1939,7 +1892,7 @@ def config(self, serder, estOnly=None, doNotDelegate=None): def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None, check=False): + firner=None, dater=None, eager=False, local=True, check=False): """ Not an inception event. Verify event serder and indexed signatures in sigers and update state @@ -1963,6 +1916,15 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, dater (Dater | None): Dater instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote check (bool): True means do not update the database in any non-idempotent way. Useful for reinitializing the Kevers from a persisted KEL without updating non-idempotent first seen .fels @@ -1980,6 +1942,7 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, self.prefixer.qb64, ked)) + local = True if local else False sner = serder.sner # Number instance ensures whole number for sequence number ilk = serder.ilk # ked["t"] @@ -1995,35 +1958,28 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # returned sigers and wigers are verified signatures # If does not validate then escrows as needed and raises ValidationError - sigers, delegator, wigers = self.valSigsDelWigs(serder=serder, - sigers=sigers, - verfers=serder.verfers, - tholder=tholder, - wigers=wigers, - toader=toader, - wits=wits, - delseqner=delseqner, - delsaider=delsaider) - - - # rotation so check rotation threshold against exposed sigers versus - # prior next digers in .ndigers - #ondices = self.exposeds(sigers) - #if not self.ntholder.satisfy(indices=ondices): - #self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) - #if delseqner and delsaider: # save in case not attached later - #self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) - #raise MissingSignatureError(f"Failure satisfying nsith=" - #f"{self.ntholder.sith} on sigs=" - #f"{[siger.qb64 for siger in sigers]}" - #f" for evt={serder.ked}.") + sigers, wigers, delpre, delseqner, delsaider = self.valSigsWigsDel( + serder=serder, + sigers=sigers, + verfers=serder.verfers, + tholder=tholder, + wigers=wigers, + toader=toader, + wits=wits, + delseqner=delseqner, + delsaider=delsaider, + eager=eager, + local=local) - # .validateSigsDelWigs above ensures thresholds met otherwise raises exception + + # .valSigWigsDel above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen - fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, wits=wits, - first=True if not check else False, seqner=delseqner, saider=delsaider, - firner=firner, dater=dater) + fn, dts = self.logEvent(serder=serder, sigers=sigers, wigers=wigers, + wits=wits, + first=True if not check else False, + seqner=delseqner, saider=delsaider, + firner=firner, dater=dater, local=local) # nxt and signatures verify so update state self.sner = sner # sequence number Number instance @@ -2052,10 +2008,10 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, raise ValidationError("Unexpected non-establishment event = {}." "".format(serder.ked)) - for k in IXN_LABELS: - if k not in ked: - raise ValidationError("Missing element = {} from {} event." - " evt = {}.".format(k, Ilks.ixn, ked)) + #for k in IXN_LABELS: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event." + #" evt = {}.".format(k, Ilks.ixn, ked)) if not sner.num == (self.sner.num + 1): # sn not in order raise ValidationError("Invalid sn = {} expecting = {} for evt " @@ -2071,13 +2027,16 @@ def update(self, serder, sigers, wigers=None, delseqner=None, delsaider=None, # Validates signers, delegation if any, and witnessing when applicable # If does not validate then escrows as needed and raises ValidationError - sigers, delegator, wigers = self.valSigsDelWigs(serder=serder, - sigers=sigers, - verfers=self.verfers, - tholder=self.tholder, - wigers=wigers, - toader=self.toader, - wits=self.wits) + sigers, wigers, delpre, _, _ = self.valSigsWigsDel( + serder=serder, + sigers=sigers, + verfers=self.verfers, + tholder=self.tholder, + wigers=wigers, + toader=self.toader, + wits=self.wits, + eager=eager, + local=local) # .validateSigsDelWigs above ensures thresholds met otherwise raises exception # all validated above so may add to KEL and FEL logs as first seen @@ -2181,67 +2140,85 @@ def rotate(self, serder): raise ValidationError(f"Invalid sith = {serder.tholder} for keys = " f"{keys} for evt = {ked}.") - - # compute wits from existing .wits with new cuts and adds from event # use ordered set math ops to verify and ensure strict ordering of wits # cuts and add to ensure that indexed signatures on indexed witness # receipts work + wits, cuts, adds = self.deriveBacks(serder) + + toader = serder.bner # Number(num=ked["bt"]) # auto converts hex num to int + if wits: + if toader.num < 1 or toader.num > len(wits): # out of bounds toad + raise ValueError(f"Invalid toad = {toader.num} for backers " + f"(wits)={wits} for event={ked}.") + else: + if toader.num != 0: # invalid toad + raise ValueError(f"Invalid toad = {toader.num} for backers " + "(wits)={wits} for event={ked}.") + + return tholder, toader, wits, cuts, adds + + + def deriveBacks(self, serder): + """Derives and return tuple of (wits, cuts, adds) for backers given + current set and any changes provided by serder. + + Returns: + wca (tuple): of + wits (list[str]): prefixes of witnesses full list (backers) + cuts (list[str]): prefixes of witnesses removed in latest est evt + adds (list[str]): prefixes of witnesses added in latest est evt + + Parameters: + serder (SerderKeri): instance of current event + """ + if serder.ilk not in (Ilks.rot, Ilks.drt) or self.sn >= serder.sn: # no changes + return self.wits, self.cuts, self.adds + witset = oset(self.wits) - cuts = serder.cuts # ked["br"] + cuts = serder.cuts cutset = oset(cuts) if len(cutset) != len(cuts): - raise ValidationError("Invalid cuts = {}, has duplicates for evt = " - "{}.".format(cuts, ked)) + raise ValidationError("Invalid cuts = {cuts}, has duplicates for " + f"evt = {serder.ked}.") if (witset & cutset) != cutset: # some cuts not in wits - raise ValidationError("Invalid cuts = {}, not all members in wits" - " for evt = {}.".format(cuts, ked)) + raise ValidationError(f"Invalid cuts = {cuts}, not all members in " + f"wits for evt = {serder.ked}.") - adds = serder.adds # ked["ba"] + adds = serder.adds addset = oset(adds) if len(addset) != len(adds): - raise ValidationError("Invalid adds = {}, has duplicates for evt = " - "{}.".format(adds, ked)) + raise ValidationError(f"Invalid adds = {adds}, has duplicates for " + f"evt = {serder.ked}.") if cutset & addset: # non empty intersection - raise ValidationError("Intersecting cuts = {} and adds = {} for " - "evt = {}.".format(cuts, adds, ked)) + raise ValidationError(f"Intersecting cuts = {cuts} and adds = {adds}" + f"for evt = {serder.ked}.") if witset & addset: # non empty intersection - raise ValidationError("Intersecting wits = {} and adds = {} for " - "evt = {}.".format(self.wits, adds, ked)) + raise ValidationError(f"Intersecting wits = {self.wits} and adds =" + f"{adds} for evt = {serder.ked}.") wits = list((witset - cutset) | addset) if len(wits) != (len(self.wits) - len(cuts) + len(adds)): # redundant? - raise ValidationError("Invalid member combination among wits = {}, cuts ={}, " - "and adds = {} for evt = {}.".format(self.wits, - cuts, - adds, - ked)) - - toader = serder.bner # Number(num=ked["bt"]) # auto converts hex num to int - if wits: - if toader.num < 1 or toader.num > len(wits): # out of bounds toad - raise ValueError(f"Invalid toad = {toader.num} for backers " - f"(wits)={wits} for event={ked}.") - else: - if toader.num != 0: # invalid toad - raise ValueError(f"Invalid toad = {toader.num} for backers " - "(wits)={wits} for event={ked}.") + raise ValidationError(f"Invalid member combination among wits = " + f"{self.wits}, cuts ={cuts}, and adds = " + f"{adds,} for evt = {serder.ked}.") + return (wits, cuts, adds) - return tholder, toader, wits, cuts, adds - def valSigsDelWigs(self, serder, sigers, verfers, tholder, - wigers, toader, wits, - delseqner=None, delsaider=None): + def valSigsWigsDel(self, serder, sigers, verfers, tholder, + wigers, toader, wits, *, + delseqner=None, delsaider=None, eager=False, + local=True): """ - Returns triple (sigers, delegator, wigers) where: + Returns triple (sigers, wigers, delegator) where: sigers is unique validated signature verified members of inputed sigers - delegator is qb64 delegator prefix if delegated else None wigers is unique validated signature verified members of inputed wigers + delegator is qb64 delegator prefix if delegated else None Validates sigers signatures by validating indexes, verifying signatures, and validating threshold sith. @@ -2265,6 +2242,15 @@ def valSigsDelWigs(self, serder, sigers, verfers, tholder, If this event is not delegated then seqner is ignored delsaider (Saider | None): instance of of delegating event said. If this event is not delegated then saider is ignored + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ if len(verfers) < tholder.size: @@ -2273,6 +2259,20 @@ def valSigsDelWigs(self, serder, sigers, verfers, tholder, [verfer.qb64 for verfer in verfers], serder.ked)) + # Filters sigers to remove any signatures from locally membered groups + # when not local (remote) event source. So that attacker can't source + # remotely compromised but locally membered signatures to satisfy threshold. + if not local and self.locallyMembered(): # is this Kever's pre a local group + if indices := self.locallyContributedIndices(verfers): + for siger in list(sigers): # copy so clean del on original elements + if siger.index in indices: + sigers.remove(siger) + if self.cues: + self.cues.push(dict(kin="remoteMemberedSig", + serder=serder, + index=siger.index)) + + # get unique verified sigers and indices lists from sigers list sigers, indices = verifySigs(raw=serder.raw, sigers=sigers, verfers=verfers) # sigers now have .verfer assigned @@ -2282,66 +2282,141 @@ def valSigsDelWigs(self, serder, sigers, verfers, tholder, raise ValidationError("No verified signatures for evt = {}." "".format(serder.ked)) - werfers = [Verfer(qb64=wit) for wit in wits] # get witnes signatures + # at least one valid controller signature so we can now escrow event + # Misfit check events that must be locally sourced (protected). + # Misfit escrow process may repair (upgrade) the protection when appropriate + # Put all misfit checks up front so never goes into any escrow but + # misfit if fails any misfit checks. + # get delegator's delpre if any for misfit check + if serder.ilk == Ilks.dip: # dip so delpre in event "di" field + delpre = serder.delpre # delegator from dip event + if not delpre: # empty or None + raise ValidationError(f"Empty or missing delegator for delegated" + f" inception event = {serder.ked}.") + elif serder.ilk == Ilks.drt: # serder.ilk == Ilks.drt so rotation + delpre = self.delpre # delpre in kever state + else: # not delegable event icp, rot, ixn + delpre = None + + # Misfit escrow checks + if (not local and (self.locallyOwned() or + self.locallyWitnessed(wits=wits) or + self.locallyDelegated(pre=delpre))): + self.escrowMFEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + raise MisfitEventSourceError(f"Nonlocal source for locally owned or" + f"locally witnessed or locally delegated" + f"event={serder.ked}, local aids=" + f"{self.prefixes}, {wits=}, " + f"delgator={delpre}.") + + werfers = [Verfer(qb64=wit) for wit in wits] # get witness public key verifiers # get unique verified wigers and windices lists from wigers list wigers, windices = verifySigs(raw=serder.raw, sigers=wigers, verfers=werfers) # each wiger now has added to it a werfer of its wit in its .verfer property - # escrow if not fully signed vs threshold + # escrow if not fully signed vs signing threshold + pre = self.prefixer.qb64 if not tholder.satisfy(indices): # at least one but not enough - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) - if delseqner and delsaider: - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) - msg=(f"Failure satisfying sith = {tholder.sith} " - f"on sigs {[siger.qb64 for siger in sigers]} " - f"for evt = {serder.said}") + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = (f"AID {pre[:4]}...{pre[-4:]}: Failure satisfying sith = {tholder.sith} " + f"on sigs {[siger.qb64 for siger in sigers]} " + f"for evt = {serder.said}") logger.trace(msg) logger.trace("Event Body=\n%s\n", serder.pretty()) raise MissingSignatureError(msg) + + # escrow if not fully signed vs prior next rotation threshold if serder.ilk in (Ilks.rot, Ilks.drt): # rotation so check prior next threshold # prior next threshold in .ntholder and digers in .ndigers ondices = self.exposeds(sigers) if not self.ntholder.satisfy(indices=ondices): - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) - if delseqner and delsaider: # save in case not attached later - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) - raise MissingSignatureError(f"Failure satisfying prior nsith=" - f"{self.ntholder.sith} with exposed " - f"sigs= {[siger.qb64 for siger in sigers]}" - f" for new est evt={serder.ked}.") - - - delegator = self.validateDelegation(serder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider) - - # Kevery .process event logic does not prevent this from seeing event when - # not local and event pre is own pre - if not self.locallyOwned(serder.pre): # not in self.prefixes - if ((wits and not self.prefixes) or # in promiscuous mode so assume must verify toad - (wits and self.prefixes and not self.local and # not promiscuous nonlocal - not (oset(self.prefixes) & oset(wits)))): # own prefix is not a witness - # validate that event is fully witnessed - - if wits: + self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider,local=local) + msg = ( + f"AID {pre[:4]}...{pre[-4:]}: Failure satisfying prior nsith = {self.ntholder.sith} " + f"with exposed sigs {[siger.qb64 for siger in sigers]} " + f"for new est evt={serder.said}") + logger.trace(msg) + logger.trace("Event Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) + + # this point the sigers have been verified and the wigers have been verified + # even if locallyOwned or locallyMembered or locallyWitnessed. + # But the toad has not yet been verified. + + if not wits: + if toader.num != 0: # bad toad non-zero and not witnessed bad event + raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") + + else: # wits so may need to validate toad + # The toad is only verified against the wigers as fully witnessed if + # not (locallyOwned or locallyMembered or locallyWitnessed) + if (not (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits))): + if wits: # is witnessed if toader.num < 1 or toader.num > len(wits): # out of bounds toad raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") - else: + else: # not witnessed if toader.num != 0: # invalid toad raise ValidationError(f"Invalid toad = {toader.num} for wits = {wits}") if len(windices) < toader.num: # not fully witnessed yet if self.escrowPWEvent(serder=serder, wigers=wigers, sigers=sigers, - seqner=delseqner, saider=delsaider): + seqner=delseqner, saider=delsaider, + local=local): # cue to query for witness receipts self.cues.push(dict(kin="query", q=dict(pre=serder.pre, sn=serder.snh))) - msg = (f"Failure satisfying toad={toader.num} for event={serder.said}\n" - f"on witness sigs=\n{[siger.qb64 for siger in wigers]}") - logger.trace(msg) - logger.trace("Event Body=\n%s\n", serder.pretty()) + msg = (f"AID {pre[:4]}...{pre[-4:]}: Failure satisfying toad={toader.num} " + f"on witness sigs {[siger.qb64 for siger in wigers]} " + f"for event={serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) raise MissingWitnessSignatureError(msg) - return sigers, delegator, wigers + + + # Delegator approves delegation by attaching valid source + # seal and reprocessing event which shows up here as delseqner, delsaider. + # Won't get to here if not local and locallyDelegated(delpre) misfit + # checks above will send nonlocal sourced delegable event to + # misfit escrow first. Mistfit escrow must first promote to local and + # reprocess event before we get to here. Assumes that attached source + # seal in this case can't be malicious since sourced locally. + # Doesn't get to here until fully signed and witnessed. + + if serder.ilk in (Ilks.dip, Ilks.drt) and self.locallyDelegated(delpre) and not self.locallyOwned(): # local delegator + # must be local if locallyDelegated or caught above as misfit + if delseqner is None or delsaider is None: # missing delegation seal + # so escrow delegable. So local delegator can approve OOB. + # and create delegator event with valid event seal of this + # delegated event and then reprocess event with attached source + # seal to delegating event, i.e. delseqner, delsaider. + self.escrowDelegableEvent(serder=serder, sigers=sigers, + wigers=wigers, local=local) + msg = f"Missing approval for delegation by {delpre} of event = {serder.said}" + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegableApprovalError(msg) + + # validateDelegation returns (None, None) when delegation validation + # does not apply. Raises ValidationError if validation applies but + # does not validate. + delseqner, delsaider = self.validateDelegation(serder, + sigers=sigers, + wigers=wigers, + wits=wits, + delpre=delpre, + delseqner=delseqner, + delsaider=delsaider, + eager=eager, + local=local) + + return (sigers, wigers, delpre, delseqner, delsaider) + + def exposeds(self, sigers): @@ -2391,9 +2466,13 @@ def exposeds(self, sigers): return odxs - def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsaider=None): + def validateDelegation(self, serder, sigers, wigers, wits, delpre, *, + delseqner=None, delsaider=None, eager=False, local=True): """ Returns delegator's qb64 identifier prefix if validation successful. + Assumes that local vs remote source checks have been applied before + this function is called. + Rules: If event is not a delegated event then not valid delegation If delegatee's own event (.mine) then valid delegation @@ -2405,21 +2484,139 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai Location Seal is from Delegate's establishment event Assumes state setup + self is last accepted event if any or yet to be accepted event, + serder is received event + Parameters: serder (SerderKERI): instance of delegated event serder - sigers (list): of Siger instances of indexed controller sigs of + sigers (list[Siger]): of Siger instances of indexed controller sigs of delegated event. Assumes sigers is list of unique verified sigs - wigers (list | None): of optional Siger instance of indexed witness sigs of + wigers (list[Siger]): of optional Siger instance of indexed witness sigs of delegated event. Assumes wigers is list of unique verified sigs + wits (list[str]): of qb64 non-transferable prefixes of witnesses used to + derive werfers for wigers + delpre (str): qb64 prefix of delegator delseqner (Seqner | None): instance of delegating event sequence number. If this event is not delegated then ignored delsaider (Saider | None): instance of of delegating event digest. If this event is not delegated ignored + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + eager (bool): True means try harder to validate event by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote Returns: - (str | None): qb64 delegator prefix or None if not delegated - - Superseding Recovery + None + + Process Logic: + A delegative event is processed differently for each of four different + parties, namely, controller of event, witness to controller of event, + delegator of event , and validator of event that is not controller, + witness or delegator. Events are processed as either local (protected) + or remote. A local event may assume that the event only came via a + protected transmission path. This might be because the event is + functions locally on a device under the supervision of the controller + or was received via some protected channel using some form of MFA. + A remote event is received in an unprotected manner. The purpose of + local and remote is to allow increased security on local events where + a threshold structure is imposed. + + A witness pool may act a threshold structure for enchanced security + when each witness only accepts local events that are protected + by a unique authentication factor thereby making the controller's + signature(s) the first factor and the set of unique witness factors + a secondary threshold factor. An attacker therefore has to compromise + not merely the controller's private key(s) but also the unique second factor + on each of a threshold satisfycing number of witnesses. + + Likewise a delegator may act as a threshold structure for enhanced security + when the delegator only accepts local events for delegation that are + protected by a unique authentication factor thereby making the + controller's signatures the first factor, a threshold satisfycing + number of unique witness factors the second layer of factors, and + the delegator's unique authentication factor as the third factor. + An attacker therefore has to compromise not merely the controller's + private key(s) but also the unique second factor + on each of a threshold satisfycing number of witnesses and the unique + third factor for the delegator. + + Controller as delegatee must accept its own delegated event prior + to full witnessing or delegator approval (anchored seal) by signing the + event in order to trigger the logic to get witness receipts and + delegator approval. This means a local (protected) event may be + accepted into controller's KEL when fully signed by controller. + + Witness must accept a controller's delegated event it witnesses prior to + full witnessing or delegator approval in order to trigger its + witnessing logic. This means a local (protected) event may be + accepted into a witness' KEL when fully signed by its controller. + + Delegator may escrow or sandbox a delegated event prior to it anchoring + a seal of the event in its KEL in order to trigger its approval logic. + Alternatively the approval logic may be triggered immediately after + it is received and authenticated on it its local (protected) channel + but before it is submitted to its local Kevery for processing. + + The delegator MUST NOT accept a delegable event unless it is locally + sourced, fully signed by its controller, and fully witnessed by its + controller's designated witness pool. + A Delegator may impose additional validation logic prior to approval. + The approval logic may be handled by an escrow that only runs if + the delegable event is sourced as local. This may require a + sandboxed kel for the delegatee in order to not corrupt its pristine + copy of the delegatee's KEL with a valid delegable event from a + malicious source. The sandboxing logic may create a virtual + delegation event with seal for the purpose of checking the delegated + event superseding logic prior to acceptance. + + A malicious attacker that compromises the pre-rotated keys of the + delegatee may issue a rotation that changes its witness pool in order + to bypass the local security logic of the witness pool. The approval + logic of the delegator may choose to not automatically approve a + delegable rotation event unliess the change to the witness pool is + below the threshold. + + The logic for superseded events is NOT a requirement for acceptance in + either a delegated event controller's KEL or its witness' KEL. The + delegator's kel creates a virtual (provisional) delegating interaction + event in order to evaluate correct superseding logic so as not to + accept an invalid supderseding delegated event into its local copy + of the delegated KEL. This virtual event is needed because superseding + logic requires an anchoring seal be present before the rules can + be fully evaluated. + + Should the actual anchor be via a superseding rotation in the + delegator's KEL not via an interaction event then the delegator must + check the logic for a virtual delegating rotation instead. + In either case the delegated event does not change so the virtual + delegating checks are sufficient to accept the delegated event + into the delegator's local copy of the delegatee's KEL. + + + Any of delegated controller, delegated witness, or delegator + of delegated event may after the fact fully validate event by + processing it as a remote event. + Then the logic applied is same as validator below. + + A validator of a delegated event that is not the event's controller, + witness, or delegator must not accept the event until is is fully + signed by the controller (threshold), fully witnessed by the witness + pool (threshold) and its seal anchored in the delegator's KEL. The + rules for event superseding in the delegated controller's kel must + also be satisfied. The logic should be the same for both local and + remote event because the validator is not one of the protected parties + to the event. + + Superseding Recovery: Supersede means that after an event has already been accepted as first seen into a KEL that a different event with the same sequence number is accepted @@ -2427,233 +2624,478 @@ def validateDelegation(self, serder, sigers, wigers=None, delseqner=None, delsai events signed by compromised keys. The result of superseded recovery is that the KEL is forked at the sn of the superseding event. All events in the superseded branch of the fork still exist but, by virtue of being superseded, - are disputed. The set of superseding events in the superseding fork forms the authoritative - branch of the KEL. All the already seen superseded events in the superseded fork + are disputed. The set of superseding events in the superseding fork forms + the authoritative branch of the KEL. + + All the already seen superseded events in the superseded fork still remain in the KEL and may be viewed in order of their original acceptance because the database stores all accepted events in order of acceptance and denotes this order using the first seen ordinal number, fn. - The fn is not the same as the sn (sequence number). + + Recall that the fn is not the same as the sn (sequence number). Each event accepted into a KEL has a unique fn but multiple events due to recovery forks may share the same sn. - Superseding Rules for Recovery at given SN (sequence number) + Superseding Rules for Recovery at given 'sn' (sequence number) - A0. Any rotation event may supersede an interaction event at the same sn. (existing rule) - A1. A non-delegated rotation may not supersede another rotation at the same sn. (modified rule) + A0. Any rotation event may supersede an interaction event at the same sn. + where that interaction event is not before any other rotation event. + (existing rule) + A1. A non-delegated rotation may not supersede another rotation at the + same sn. (modified rule) A2. An interaction event may not supersede any event. ( existing rule). (B. and C. below provide the new rules) - B. A delegated rotation may supersede another delegated rotation at the same sn - under either of the following conditions: + B. A delegated rotation may supersede the latest seen delegated rotation + at the same sn under either of the following conditions: + B1. The superseding rotation's delegating event is later than - the superseded rotation's delegating event in the delegator's KEL, i.e. the - sn of the superseding event's delegation is higher than the superseded event's - delegation. - B2. The sn of the superseding rotation's delegating event is the same as - the sn of the superseded rotation's delegating event in the delegator's KEL - and the superseding rotation's delegating event is a rotation and the - superseded rotation's delegating event is an interaction, - i.e. the superseding rotation's delegating event is itself a superseding - rotation of the superseded rotations delegating interaction event in the - delgator's KEL + the superseded rotation's delegating event in the delegator's KEL, + i.e. the sn of the superseding event's delegation is higher than + the superseded event's delegation. + + B2. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL both have the same delegating event and the index + of the delegating seal of the superceding delegation seal is later + (higher) than the index of the delegating seal of the supderseded + rotation. In other words the delegating event is the same event but + the superseding delegation seal appears later in the seal list of the + delegating event than the superseded delegation seal. + + B3. The sn of the superseding rotation's delegating event is the + same as the sn of the superseded rotation's delegating event in the + delegator's KEL and the superseding rotation's delegating event is + a rotation and the superseded rotation's delegating event is an + interaction, i.e. the delegating events are not the same event and + the superseding rotation's delegating event is itself a superseding + rotation of the superseded rotations delegating interaction event + in the delgator's KEL. C. IF Neither A nor B is satisfied, then recursively apply rules A. and B. to - the delegating events of those delegating events and so on until either A. or B. - is satisfied, or the root KEL of the delegation has been reached. - C1. If neither A. nor B. is satisfied by recursive application on the - delegator's KEL (i.e. the root KEL of the delegation has been reached without - satisfaction) then the superseding rotation is discarded. The terminal case of - the recursive application will occur at the root KEL which by defintion is - non-delegated wherefore either A. or B. must be satisfied, or else the - superseding rotation must be discarded. - - """ - if serder.ilk not in (Ilks.dip, Ilks.drt): # not delegated - return None # delegator is None - - # verify delegator and attachment pointing to delegating event - if serder.ilk == Ilks.dip: - delegator = serder.delpre # delegator from dip event - if not delegator: - raise ValidationError(f"Empty or missing delegator for delegated" - f" inception event = {serder.ked}.") - else: # serder.ilk == Ilks.drt so rotation - delegator = self.delegator + the delegating events of those delegating events and so on until + either A. or B. is satisfied, or the root KEL of the delegation + which must be undelegated has been reached. + + C1. If neither A. nor B. is satisfied by recursive application on the + delegator's KEL (i.e. the root KEL of the delegation has been reached + without satisfaction) then the superseding rotation is discarded. + The terminal case of the recursive application will occur at the + root KEL which by defintion is non-delegated wherefore either + A. or B. must be satisfied, or else the superseding rotation must + be discarded. + + Note: The latest seen delegated rotation constraint means that any earlier + delegated rotations CAN NOT be superseded. This greatly simplifies the + validation logic and avoids a potential infinite regress of forks in the + delegated identifier's KEL while allowing the delegate to + detect that a compromised delegation has occurred and give an + opportunity for the delegator to refuse to approve a subsequent + delegated rotation without additional verification with the delegate + that the subsequent delegated rotation was not compromised. + + In order to capture control of a delegated identifier the attacker must + issue a delegated rotation that rotates to keys under the control of the + attacker that must be approved by the delegator. A recovery rotation must + therefore supersede the compromised rotation. If the attacker is able + to issue and get approved by the delegator a subsequent rotation + that follows but does not supersede the compromising rotation then + recovery is no longer possible because the delegatee would no longer + control the privete keys needed to verifiably sign a recovery rotation. + + One way that detectability may be assured is when the delegator imposes + a minimum time between approvals of a delegated rotation that is + sufficient for the delgate to detect a compromised rotation recovery. + Attempts to rotate sooner than the minimum time since the immediately + prior rotation are refused until further verification has occurred. + + A delegated rotation that occurs after the minimum time since the + immediately prior delegated rotation might be automatically approved + to minimize latency. While a subsequent delegated rotation that occurs + within the minimum time would not be approed to maximize safety. + The minimum time window is designed to give the delegate enough time + to detect a comprimised or duplicitious superseding rotation and + prevent the additional verification from proceding. + + ToDo: + + Repair the approval source seal couple in the 'aess' database on recursive + climb the kel tree. Once an event has been accepted into its kel. + Later adding a source seal couple to 'aes' should then be OK from a + security perspective since its only making discovery less expensive. + + When malicious source seal couple is received but event is validly + delegated and the delegation source seal is repaired then need to replace + malicious source seal couple with repaired seal so repaired seal not + malicous seal gets written to 'aes' db. When the event is valid but + non-delegated then need to nullify malicous source seal couple so it + does not get written to 'aes' datable + """ + if not delpre: # not delegable so no delegation validation needed + return (None, None) # non-delegated so delseqner delsaider must be None # if we are the delegatee, accept the event without requiring the # delegator validation via an anchored delegation seal or by requiring - # it to be witnessed - # ToDo XXXX add local lax check after figure out dist multisig group - # ToDo XXXX add check for witness to accept so that witness will - # add to its KEL without waiting for delegation seal to be anchored in - # delegator's KEL witness cue in Kevery will then generate reciept - if self.locallyOwned(serder.pre) or self.locallyWitnessed(serder=serder): - return delegator - - # during initial delegation we just escrow the delcept event - if delseqner is None and delsaider is None and delegator is not None: - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) - msg = (f"No delegation seal for delegator {delegator} with event = {serder.said}") + # it to be witnessed. Query witness cue in Kevery will then request witnessing. + # Controller accepts without waiting for witnessor nor for the delegation + # seal to be anchored in delegator's KEL. + # Witness accepts without waiting for delegation seal to be anchored in + # delegator's KEL. Witness cue in Kevery will then generate receipt + if (self.locallyOwned() or self.locallyMembered() or + self.locallyWitnessed(wits=wits)): + return (None, None) # not validated so delseqner delsaider must be None + + if self.kevers is None or delpre not in self.kevers: # missing delegator KEL + # ToDo XXXX cue a trigger to get the KEL of the delegator. This may + # require OOBIing with the delegator. + # The processPDEvent should also cue a trigger to get KEL + # of delegator if still missing when processing escrow later. + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = f"Missing KEL of delegator {delpre} of evt {serder.sn} {serder.ilk} {serder.said}" + logger.info(msg) logger.debug("Event Body=\n%s\n", serder.pretty()) raise MissingDelegationError(msg) - ssn = validateSN(sn=delseqner.snh, inceptive=False) # delseqner Number should already do this - #ssn = sner.num sner is Number seqner is Seqner need to replace Seqners with Numbers - - # get the dig of the delegating event. Using getKeLast ensures delegating - # event has not already been superceded - key = snKey(pre=delegator, sn=ssn) # database key - raw = self.db.getKeLast(key) # get dig of delegating event - - if raw is None: # no delegating event at key pre, sn - # ToDo XXXX create cue to send query to fetch delegating event from - # delegator - self.cues.push(dict(kin="query", q=dict(pre=delegator, - sn=delseqner.snh, - dig=delsaider.qb64))) - - # escrow event here - inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False - sn = validateSN(sn=serder.snh, inceptive=inceptive) - self.escrowPSEvent(serder=serder, sigers=sigers, wigers=wigers) - self.escrowPACouple(serder=serder, seqner=delseqner, saider=delsaider) - raise MissingDelegationError("No delegating event from {} at {} for " - "evt = {}.".format(delegator, - delsaider.qb64, - serder.ked)) - - # get the delegating event from dig - ddig = bytes(raw) - key = dgKey(pre=delegator, dig=ddig) # database key - raw = self.db.getEvt(key) - if raw is None: # drop event - raise ValidationError("Missing delegation from {} at event dig = {} for evt = {}." - "".format(delegator, ddig, serder.ked)) - - dserder = serdering.SerderKERI(raw=bytes(raw)) # delegating event - # compare digests to make sure they match here - if not dserder.compare(said=delsaider.qb64): # drop event - raise ValidationError("Invalid delegation from {} at event dig = {} for evt = {}." - "".format(delegator, ddig, serder.ked)) - - if self.kevers is None or delegator not in self.kevers: # drop event - raise ValidationError("Missing Kever for delegator = {} for evt = {}." - "".format(delegator, serder.ked)) - - dkever = self.kevers[delegator] - if dkever.doNotDelegate: # drop event - raise ValidationError("Delegator = {} for evt = {}," - " does not allow delegation.".format(delegator, - serder.ked)) - - - found = False # find event seal of delegated event in delegating data - # XXXX ToDo need to change logic here to support native CESR seals not just dicts - # for JSON, CBOR, MGPK - for dseal in dserder.seals: # find delegating seal anchor - if tuple(dseal.keys()) == SealEvent._fields: - seal = SealEvent(**dseal) - if (seal.i == serder.pre and - seal.s == serder.sner.numh and - serder.compare(said=seal.d)): - found = True - break - - if not found: # drop event - raise ValidationError("Missing delegation from {} in {} for evt = {}." - "".format(delegator, dserder.seals, serder.ked)) - - # Assumes database is reverified each bootup chain-of-custody of dic broken. + + dkever = self.kevers[delpre] # get delegator's KEL + if dkever.doNotDelegate: # drop event if delegation not allowed + msg = (f"Delegator {delpre} does not allow delegation on evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) + + dserder = None # no delegation event yet + if delseqner is None or delsaider is None: # missing delegation seal ref + if eager: # walk kel here to find + seal = dict(i=serder.pre, s=serder.snh, d=serder.said) + dserder = self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal) + if dserder is not None: # found seal in dserder + delseqner = coring.Seqner(sn=dserder.sn) # replace with found + delsaider = coring.Saider(qb64=dserder.said) # replace with found + + if not dserder: # just escrow and try later + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = (f"No delegation seal for delegator {delpre} on evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) + + if delseqner and delsaider and not dserder: # given couple not found + # ToDo XXXX need to replace Seqners with Numbers + # Get delegating event from delseqner and delpre + ssn = Number(num=delseqner.sn).validate(inceptive=False).sn + # get the dig of the delegating event. Using getKeLast ensures delegating + # event has not already been superceded + key = snKey(pre=delpre, sn=ssn) # database key + # get dig of last delegating event purported at sn + raw = self.db.getKeLast(key) # last means not disputed or superseded + if raw is None: # no delegatint event yet. no index at key pre, sn + # Have to wait until delegating event at sn shows up in kel + # ToDo XXXX process this cue of query to fetch delegating event from + # delegator + self.cues.push(dict(kin="query", q=dict(pre=delpre, + sn=delseqner.snh, + dig=delsaider.qb64))) + # escrow event here + inceptive = True if serder.ilk in (Ilks.icp, Ilks.dip) else False + sn = Number(num=serder.sn).validate(inceptive=inceptive).sn + # if we were to allow for malicous local delegator source seal then we + # must check for locallyOwned(delpre) first and escrowDelegable. + # otherwise escrowPDEvent + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) + + # get the latest delegating event candidate from dig given by pre,sn index + ddig = bytes(raw) + key = dgKey(pre=delpre, dig=ddig) # database key + raw = self.db.getEvt(key) # get actual last event + if raw is None: # drop event should never happen unless database is broken + msg = (f"Missing delegation from {delpre} at event dig = {ddig} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) + + dserder = serdering.SerderKERI(raw=bytes(raw)) # purported delegating event + + found = False # find event seal of delegated event in delegating data + # search purported delegating event from source seal couple + + for dseal in dserder.seals: # find delegating seal anchor + # XXXX ToDo need to change logic here to support native CESR seals not just dicts + # for JSON, CBOR, MGPK + if tuple(dseal) == SealEvent._fields: + dseal = SealEvent(**dseal) + if (dseal.i == serder.pre and + dseal.s == serder.sner.numh and + serder.compare(said=dseal.d)): + found = True + break + + if not found: # nullify and escrow to try harder later + # worst case assume source seal was malicious so nullify it and + # attempt to repair by escrowing and eager search later + delseqner = delsaider = None # nullify + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = (f"No delegation seal for delegator {delpre} of evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) + + ## Found valid anchoring seal of delegator delpre + ## compare saids to ensure match of delegating event and source seal + ## repair source seal to match last event at sn in source seal. + ## Original delegating event in seal may have been disputed or + ## superseded, but if latest matches then we want to repair + #if not dserder.compare(said=delsaider.qb64): # drop event + #raise ValidationError(f"Invalid delegation from {delpre} at event" + #f" dig={ddig} for evt={serder.ked}.") + + delseqner = Seqner(snh=dserder.snh) # replace with found + delsaider = Saider(qb64=dserder.said) # replace with found + + # Since found valid anchoring seal so can confirm delegation successful + # unless its one of the superseding conditions. + # Valid if not superseding drt of drt. + # Assumes database is reverified each bootup or otherwise when + # chain-of-custody of disc has been broken. # Rule for non-supeding delegated rotation of rotation. - # Returning delegator indicates success and eventually results acceptance + # Returning delegator indicates success and eventually results in acceptance # via Kever.logEvent which also writes the delgating event source couple to # db.aess so we can find it later - if ((serder.ilk == Ilks.dip) or # delegated inception - (serder.sner.num == self.sner.num + 1) or # inorder event - (serder.sner.num == self.sner.num and - self.ilk == Ilks.ixn and - serder.ilk == Ilks.drt)): # recovery rotation superseding ixn - return delegator # indicates delegation valid with return of delegator - + if ((serder.ilk == Ilks.dip) or # superseding is delegated inception or + (serder.sner.num == self.sner.num + 1) or # superseding is inorder later or + (serder.sner.num == self.sner.num and # superseding event at same sn and + self.ilk == Ilks.ixn and # superseded is interaction and + serder.ilk == Ilks.drt)): # superseding is rotation + return (delseqner, delsaider) # indicates delegation valid + + # get to here means drt rotation superseding another drt rotation # Kever.logEvent saves authorizer (delegator) seal source couple in - # db.aess data base so can use it here to recusively look up delegating - # events + # db.aess (.getAes, .setAes etc) data base so can use it here to + # recusively look up delegating events # set up recursive search for superseding delegations + # get original potentially superseded delegation + serfo = self.serder # original accepted delegated event i.e. serf original + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, original=True, + eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) + # already have new potential superseding delegation serfn = serder # new potentially superseding delegated event i.e. serf new bossn = dserder # new delegating event of superseding delegated event i.e. boss new - serfo = self.serder # original delegated event i.e. serf original - bosso = self.fetchDelegatingEvent(delegator, serfo) while (True): # superseding delegated rotation of rotation recovery rules - # Only get to here if same sn for drt existing and drt superseding - - if (bossn.sn > bosso.sn or # later supersedes - (bossn.Ilk == Ilks.drt and - bosso.Ilk == Ilks.ixn) ): # drt supersedes ixn - return delegator # valid superseding delegation + # Only get to here if same sn for delegated event and both + # existing and superseding delegated events are rotations + if (bossn.sn > bosso.sn or # superseding delgation is later or + (bossn.Ilk == Ilks.drt and # superseding delegation is rotation and + bosso.Ilk == Ilks.ixn) ): # superseded delegation is interaction + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source if bossn.said == bosso.said: # same delegating event nseals = [SealEvent(**seal) for seal in bossn.seals - if tuple(seal.keys()) == SealEvent._fields] - nindex = nseals.index(SealEvent(i=serfn.pre, s=serfn.snh, d=serfn.said)) + if tuple(seal) == SealEvent._fields] + nindex = nseals.index(SealEvent(i=serfn.pre, + s=serfn.snh, + d=serfn.said)) oseals = [SealEvent(**seal) for seal in bosso.seals - if tuple(seal.keys()) == SealEvent._fields] - oindex = oseals.index(SealEvent(i=serfo.pre, s=serfo.snh, d=serfo.said)) + if tuple(seal) == SealEvent._fields] + oindex = oseals.index(SealEvent(i=serfo.pre, + s=serfo.snh, + d=serfo.said)) - if nindex > oindex: # later seal supersedes + if nindex > oindex: # superseding delegation seal is later # assumes index can't be None - return delegator # valid superseding delegation + # valid superseding delegation up chain so tail link valid + return (delseqner, delsaider) # tail event's delegation source - else: + else: # not superseded # ToDo: XXXX may want to cue up business logic for delegator # if self.mine(delegator): # failed attempt at recovery - raise ValidationError(f"Invalid delegation recovery rotation" - f"of {serfo.ked} by {serfn.ked}") + msg = f"Invalid delegation recovery rotation of {serfo.pre} by {serfn.pre}" + logger.info(msg) + logger.debug("Delegate Event Body=\n%s\n", serfo.pretty()) + logger.debug("Delegator Event Body=\n%s\n", serfn.pretty()) + raise ValidationError(msg) # tie condition same sn and drt so need to climb delegation chain serfn = bossn - bossn = self.fetchDelegatingEvent(delegator, serfn) + if not (bossn := self.fetchDelegatingEvent(delpre, serfn, + original=False, + eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) serfo = bosso - bosso = self.fetchDelegatingEvent(delegator, serfo) + if not (bosso := self.fetchDelegatingEvent(delpre, serfo, + original=True, + eager=eager)): + self.escrowPDEvent(serder=serder, sigers=sigers, wigers=wigers, + seqner=delseqner, saider=delsaider, local=local) + msg = (f"No delegating event from {delpre} at {delsaider.qb64} for evt " + f"{serder.sn} {serder.ilk} {serder.said}") + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise MissingDelegationError(msg) # repeat + # should never get to here + + + + def fetchDelegatingEvent(self, delpre, serder, *, original=True, eager=False): + """Returns delegating event of delegator given by its aid delpre of + delegated event given by serder otherwise raises ValidationError. + Assumes delegation event must have been accepted at some point even if + it has subsequently been superseded. Therefore only looks in .aess for + delegating event source seal references. So do not use for fetching + delegating eventsthat have yet to be accepted. + Checks that found delegation events have both been accepted. - def fetchDelegatingEvent(self, delegator, serder): - """Returns delegating event by delegator of delegated event given by - serder otherwise raises ValidationError. - Assumes serder is already delegated event + When delegated event in serder has been accepted then will repair its + .aess entry if needed. + + Returns: + dserder (SerderKERI): delegating key event with delegating seal of + delegated event serder. If can't fetch then + returns None when eager==False or raises + ValidationError if can't fetch but eager==True Parameters: - delegator (str): qb64 of identifier prefix of delegator + delpre (str): qb64 of identifier prefix of delegator serder (SerderKERI): delegated serder - + original (bool): True means delegated event is the original candidate + to be superseded. This means kel walk search should + include superseded or disputed events. + False means the delegated event is new candidate to + supersede. This means kel walk search should not + include superseded or disputed events. + eager (bool): True means do more expensive KEL walk instead of escrow + False means do not do expensive KEL walk now. + + Assumes db.aes source seal couple of delegating event cannot be written + unless that event has been accepted (first seen) into delegator's KEL + even if it has later been superseded. + Confirms this by checking the .fons database of the delegator. + + A malicious escrow of delegating event could only write source seal + couple to escrow in db.udes not in accepted db.aes + + Delegating event may have been superceded but delegated event validator + does not know it yet because db.aes keeps original delegating event + source seal from before is was superseded. + + Therefore lookup of delegating event needs to check delegating event via + db.fons and not last key event in .kels which would only return the + superseding event. + + When walking delegation tree a given delegating event may have + been superceded by another delegating event in the same delegator + KEL. This method does not distinguish between superseding and + superseded so can't assume that the delegating event is the last + event at its sn, i.e. the superseding event. + So this method must use db.fons to ensure that delegating + event was accepted (first seen) even if it has subsequently been + superseded. + + When the .aess entry is missing and eager is True and a valid delegation + is found and if delegate has been accepted then repairs missing .aess + entry. Otherwise does not repair but simply returns found delegation. + + Found delegation may not be superseding so do not repair .aess unless + delegate was already accepted. """ - dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key - if (couple := self.db.getAes(dgkey)): # delegation source couple - seqner, saider = deSourceCouple(couple) - dgkey = dgKey(pre=delegator, dig=saider) # event at its said - # get event by dig not by sn at last event because may have been superceded - if not (raw := self.db.getEvt(dgkey)): - # database broken this should never happen so do not supersede - raise ValidationError(f"Missing delegation event for {serder.ked}") + dgkey = dgKey(pre=serder.preb, dig=serder.saidb) # database key of delegate - dserder = serdering.SerderKERI(raw=bytes(raw)) # original delegating event i.e. boss original + if (couple := self.db.getAes(dgkey)): # delegation source couple at delegate + seqner, saider = deSourceCouple(couple) + deldig = saider.qb64 # dig of delegating event + # extra careful double check that .aes is valid by getting + # fner = first seen Number instance index + if not self.db.fons.get(keys=(delpre, deldig)): # Not first seen yet? + if original: # should not happen aes database broken + # repair by deleting aes and returning None so it escrows + # and then next time around find below with repair it + self.db.delAes(dgkey) # delete aes so next time repairs it + # superseding may not have happened yet so let it escrow + return None + ddgkey = dgKey(pre=delpre, dig=deldig) # database key of delegation + if not (raw := self.db.getEvt(ddgkey)): # in fons but no event + # database broken this should never happen + msg = f"Missing delegation event for {serder.said}" + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) + # original delegating event i.e. boss original + dserder = serdering.SerderKERI(raw=bytes(raw)) + return dserder - else: #try to find seal the hard way + elif eager: # missing aes but try to find seal by walking delegator's KEL seal = SealEvent(i=serder.pre, s=serder.snh, d=serder.said)._asdict - if not (dserder := self.db.findAnchoringSealEvent(pre=serder.delpre, seal=seal)): - # database broken this should never happen so do not supersede - raise ValidationError(f"Missing delegation source seal for {serder.ked}") + if original: # search all events in delegator's kel not just last + if not (dserder := self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): + # database broken this should never happen so do not validate + # since original must have been validated so it must have + # all its delegation chain. + msg = f"Missing delegation source seal for {serder.said}" + logger.info(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) + else: # only search last events in delegator's kel + if not (dserder := self.db.fetchLastSealingEventByEventSeal(pre=delpre, + seal=seal)): + # superseding delegation may not have happened yet so escrow + # ToDo XXXX need to cue up to get latest events in + # delegator's kel. + # raise ValidationError(f"Missing delegation source seal for {serder.ked}") + return None + + # Only repair .aess when found delegation is for delegated event that + # has been accepted delegated event i.e has .fons entry + if self.db.fons.get(keys=(serder.pre, serder.said)): + # Repair .aess of delegated event by writing found source + # seal couple of delegation. This is safe becaause we confirmed + # delegation event was accepted in delegator's kel. + couple = dserder.sner.huge.encode() + dserder.saidb + self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + + return dserder + + else: # not found so return None to escrow and get fixed later + return None - return dserder def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, - seqner=None, saider=None, firner=None, dater=None): + seqner=None, saider=None, firner=None, dater=None, local=True): """ Update associated logs for verified event. Update is idempotent. Logs will not write dup at key if already exists. @@ -2677,8 +3119,14 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, dater is optional Dater instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False fn = None # None means not a first seen log event so does not return an fn + dgkeys = (serder.pre, serder.said) dgkey = dgKey(serder.preb, serder.saidb) dtsb = helping.nowIso8601().encode("utf-8") self.db.putDts(dgkey, dtsb) # idempotent do not change dts if already @@ -2688,111 +3136,293 @@ def logEvent(self, serder, sigers=None, wigers=None, wits=None, first=False, self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if wits: self.db.wits.put(keys=dgkey, vals=[coring.Prefixer(qb64=w) for w in wits]) + self.db.putEvt(dgkey, serder.raw) # idempotent (maybe already excrowed) - val = (coring.Prefixer(qb64b=serder.preb), coring.Seqner(sn=serder.sn)) - for verfer in (serder.verfers if serder.verfers is not None else []): - self.db.pubs.add(keys=(verfer.qb64,), val=val) - for diger in (serder.ndigers if serder.ndigers is not None else []): - self.db.digs.add(keys=(diger.qb64,), val=val) + # update event source + + # delegation for authorized delegated or issued event + # when seqner and saider are provided they are only assured to be valid + # kever for event if kel is delegated and not locallyOwned + # and not locallyWitnessed as the validateDelegation is short circuited + # for non delegated kels, local controllers, and local witnesses. + # These checks prevent ddos via malicious source seal attachments. + # MUST NOT setAes if not delegated or locallyOwned or locallyWitnessed + if (self.delpre and not serder.ilk == Ilks.ixn and not self.locallyOwned() + and not self.locallyWitnessed(wits=wits) and seqner and saider): + couple = seqner.qb64b + saider.qb64b + self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + + #if seqner and saider: + #couple = seqner.qb64b + saider.qb64b + #self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal + + if esr := self.db.esrs.get(keys=dgkeys): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkeys, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkeys, val=esr) + + pre = self.prefixer.qb64 if first: # append event dig to first seen database in order - if seqner and saider: # delegation for authorized delegated or issued event - couple = seqner.qb64b + saider.qb64b - self.db.setAes(dgkey, couple) # authorizer (delegator/issuer) event seal fn = self.db.appendFe(serder.preb, serder.saidb) if firner and fn != firner.sn: # cloned replay but replay fn not match if self.cues is not None: # cue to notice BadCloneFN self.cues.push(dict(kin="noticeBadCloneFN", serder=serder, - fn=fn, firner=firner, dater=dater)) - logger.info("Mismatch Cloned Replay FN: First seen " - "ordinal fn %s dig %s and clone fn %s event=%s", - fn, serder.pre, firner.sn, serder.said) - logger.debug("Event Body=\n%s\n", serder.pretty()) + fn=fn, firner=firner, dater=dater)) + logger.info("Kever: Mismatch Cloned Replay FN: %s First seen " + "ordinal fn %s and clone fn %s, said=%s", + serder.preb, fn, firner.sn, serder.said) + logger.debug("Event body=\n%s\n", serder.pretty()) if dater: # cloned replay use original's dts from dater dtsb = dater.dtsb self.db.setDts(dgkey, dtsb) # first seen so set dts to now self.db.fons.pin(keys=dgkey, val=Seqner(sn=fn)) - logger.debug("Kever: First seen %s %s SAID=%s for %s at %s", - fn, serder.ilk, serder.said, serder.pre, dtsb.decode("utf-8")) + logger.debug("AID %s...%s: First seen %s at sn=%s valid event SAID=%s for %s at %s", + pre[:4], pre[-4:], serder.ilk, fn, serder.said, + serder.pre, dtsb.decode("utf-8")) logger.debug("Event Body=\n%s\n", serder.pretty()) self.db.addKe(snKey(serder.preb, serder.sn), serder.saidb) - pre = self.prefixer.qb64 - logger.info("[AID %s...%s]: Added to KEL %s at sn=%s valid event SAID=%s for AID %s", - pre[:4], pre[-4:], serder.ilk, serder.sn, serder.said, serder.pre) + logger.info("AID %s...%s: Added to KEL %s at sn=%s valid event SAID=%s", + pre[:4], pre[-4:], serder.ilk, serder.sn, serder.said) logger.debug("Event Body=\n%s\n", serder.pretty()) return (fn, dtsb.decode("utf-8")) # (fn int, dts str) if first else (None, dts str) - def escrowPSEvent(self, serder, sigers, wigers=None): + + def escrowMFEvent(self, serder, sigers, wigers=None, + seqner=None, saider=None, local=True): """ - Update associated logs for escrow of partially signed event - or fully signed delegated event but not yet verified delegation. + Update associated logs for escrow of MisFit event Parameters: - serder is SerderKERI instance of event - sigers is list of Siger instances of indexed controller sigs - wigers is optional list of Siger instance of indexed witness sigs + serder (SerderKERI): instance of event + sigers (list): of Siger instance for event + wigers (list): of witness signatures + seqner (Seqner): instance of sn of event delegatint/issuing event if any + saider (Saider): instance of dig of event delegatint/issuing event if any + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) - self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + self.db.putEvt(dgkey, serder.raw) if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) - self.db.putEvt(dgkey, serder.raw) - snkey = snKey(serder.preb, serder.sn) - self.db.addPse(snkey, serder.saidb) # b'EOWwyMU3XA7RtWdelFt-6waurOTH_aW_Z9VTaU-CshGk.00000000000000000000000000000001' - logger.debug("Kever: Escrowed partially signed or delegated " - "event %s for AID %s", serder.said, serder.pre) - logger.debug("Event Body=\n%s\n", serder.pretty()) + if seqner and saider: + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + + res = self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) + # log escrowed + logger.debug("Kever state: escrowed misfit event=\n%s\n", serder.pretty()) - def escrowPACouple(self, serder, seqner, saider): + + def escrowDelegableEvent(self, serder, sigers, wigers=None, local=True): """ - Update associated logs for escrow of partially authenticated issued event. - Assuming signatures are provided elsewhere. Partial authentication results - from either a partially signed event or a fully signed delegated event - but whose delegation is not yet verified. + Update associated logs for escrow of Delegable event that needs delegation + via an anchored seal in delegator's KEl - Escrow allows escrow processor to retrieve serder from key and source - couple from val in order to to re-verify authentication status. Sigs - are escrowed elsewhere. + Parameters: + serder (SerderKERI): instance of event + sigers (list): of Siger instance for event + wigers (list): of witness signatures + seqner (Seqner): instance of sn of event delegatint/issuing event if any + saider (Saider): instance of dig of event delegatint/issuing event if any + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False + dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + self.db.putEvt(dgkey, serder.raw) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + self.db.delegables.add(snKey(serder.preb, serder.sn), serder.saidb) + # log escrowed + logger.debug("Kever state: escrowed delegable event=\n%s\n", serder.pretty()) + + + def escrowPSEvent(self, serder, *, sigers=None, wigers=None, + seqner=None, saider=None, local=True): + """ + Update associated logs for escrow of partially signed event + or fully signed delegated event but not yet verified delegation. Parameters: - serder is SerderKERI instance of delegated or issued event + serder is SerderKERI instance of event + sigers is list of Siger instances of indexed controller sigs + wigers is optional list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer - saider is Saider instance of said of delegator/issuer + saider is Diger instance of digest of delegator/issuer + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent - logger.info("Kever state: Escrowed source couple for partially signed " - "or delegated event %s for AID %s", serder.said, serder.pre) - logger.debug("Event Body=\n%s\n", serder.pretty()) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent + if sigers: + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + + self.db.putEvt(dgkey, serder.raw) + # update event source + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + snkey = snKey(serder.preb, serder.sn) + self.db.addPse(snkey, serder.saidb) + logger.debug("Kever state: Escrowed partially signed or delegated " + "event = %s\n", serder.ked) + - def escrowPWEvent(self, serder, wigers, sigers=None, seqner=None, saider=None): + def escrowPWEvent(self, serder, *, sigers=None, wigers=None, + seqner=None, saider=None, local=True): """ Update associated logs for escrow of partially witnessed event Parameters: serder is SerderKERI instance of event - wigers is list of Siger instance of indexed witness sigs sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs seqner is Seqner instance of sn of seal source event of delegator/issuer saider is Diger instance of digest of delegator/issuer + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent - if wigers: - self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if sigers: self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent self.db.putEvt(dgkey, serder.raw) + # update event source + if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + logger.trace("Kever state: Escrowed partially witnessed event = %s", serder.said) logger.trace("Event Body=\n%s\n", serder.pretty()) return self.db.addPwe(snKey(serder.preb, serder.sn), serder.saidb) + def escrowPDEvent(self, serder, *, sigers=None, wigers=None, + seqner=None, saider=None, local=True): + """ + Update associated logs for escrow of partially delegated or otherwise + authorized issued event. + Assumes sigs (controller signatures) and wigs (witness signatures) are + provided elsewhere. Partial authentication occurs once an event is + fully signed and witnessed but the authorizing (delegating) source + seal in the authorizer's (delegator's) key has not yet been verified. + + The escrow of the authorizing event ref via source seal is not idempotent + so that malicious or erroneous source seals may be nullified and/or + replaced by found source seals. + + Escrow allows escrow processor to retrieve serder from key and source + couple from val in order to to re-verify authentication status. + + Parameters: + serder is SerderKERI instance of event + sigers is optional list of Siger instances of indexed controller sigs + wigers is list of Siger instance of indexed witness sigs + seqner is Seqner instance of sn of seal source event of delegator/issuer + saider is Diger instance of digest of delegator/issuer + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + + """ + local = True if local else False + dgkey = dgKey(serder.preb, serder.saidb) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) # idempotent + + if sigers: # idempotent + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + if wigers: # idempotent + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: # non-idempotent pin to repair replace + self.db.udes.pin(keys=dgkey, val=(seqner, saider)) # non-idempotent + logger.debug(f"Kever state: Replaced escrow source couple sn=" + f"{seqner.sn}, said={saider.qb64} for partially " + f"delegated/authorized event said={serder.said}.") + else: + self.db.udes.rem(keys=dgkey) # nullify non-idempotent + logger.debug(f"Kever state: Nullified escrow source couple for " + f"partially delegated/authorized event said=" + f"{serder.said}.") + + self.db.putEvt(dgkey, serder.raw) # idempotent + + # update event source local or remote + if (esr := self.db.esrs.get(keys=dgkey)): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + logger.debug(f"Kever state: Escrowed partially delegated event=\n" + f"{serder.ked}\n.") + return self.db.pdes.addOn(keys=serder.pre, on=serder.sn, val=serder.said) + + def state(self): """ Returns KeyStateRecord instance of current key state @@ -2824,7 +3454,7 @@ def state(self): toad=self.toader.num, wits=self.wits, cnfg=cnfg, - dpre=self.delegator, + dpre=self.delpre, ) ) @@ -2853,6 +3483,8 @@ def fetchPriorDigers(self, sn: int | None = None) -> list | None: pre = self.prefixer.qb64 if sn is None: sn = self.lastEst.s - 1 + if sn < 0: + return None for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) @@ -2902,7 +3534,7 @@ def fetchLatestContribTo(self, verfers, sn: int | None = None): for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) raw = self.db.getEvt(dgkey) - serder = serdering.SerderKERI(raw=bytes(raw)) + serder = serdering.SerderKERI(raw=bytes(raw), verify=False) if serder.estive: # establishment event key = serder.verfers[0].qb64 try: @@ -2955,7 +3587,7 @@ def fetchLatestContribFrom(self, verfer, sn: int | None = None): for digb in self.db.getKelBackIter(pre, sn): dgkey = dgKey(pre, digb) raw = self.db.getEvt(dgkey) - serder = serdering.SerderKERI(raw=bytes(raw)) + serder = serdering.SerderKERI(raw=bytes(raw), verify=False) if serder.estive: # establishment event keys = [verfer.qb64 for verfer in serder.verfers] try: @@ -3034,8 +3666,8 @@ def __init__(self, *, cues=None, db=None, rvy=None, lax (bool): True means operate in promiscuous (unrestricted) mode, False means operate in nonpromiscuous (restricted) mode as determined by local and prefixes - local (bool): True means only process msgs for own events if not lax - False means only process msgs for not own events if not lax + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation cloned (bool): True means cloned message stream so use attached datetimes from clone source not own. False means use current datetime @@ -3052,11 +3684,12 @@ def __init__(self, *, cues=None, db=None, rvy=None, self.db = db self.rvy = rvy self.lax = True if lax else False # promiscuous mode - self.local = True if local else False # local vs nonlocal restrictions + self.local = True if local else False # local vs nonlocal default self.cloned = True if cloned else False # process as cloned self.direct = True if direct else False # process as direct mode self.check = True if check else False # process as check mode + @property def kevers(self): """ @@ -3102,26 +3735,38 @@ def fetchWitnessState(self, pre, sn): def processEvent(self, serder, sigers, *, wigers=None, delseqner=None, delsaider=None, - firner=None, dater=None): + firner=None, dater=None, eager=False, local=None): """ Process one event serder with attached indexd signatures sigers Parameters: - serder is SerderKERI instance of event to process - sigers is list of Siger instances of attached controller indexed sigs - wigers is optional list of Siger instances of attached witness indexed sigs - delseqner is Seqner instance of delegating event sequence number. + serder (SerderKERI): instance of event to process + sigers (list[Siger]): instances of attached controller indexed sigs + wigers (list[Siger]|None): instances of attached witness indexed sigs + otherwise None + delseqner (Seqner|None): instance of delegating event sequence number. If this event is not delegated then seqner is ignored - delsaider is Saider instance of of delegating event SAID. + delsaider (Saider|None): instance of of delegating event SAID. If this event is not delegated then saider is ignored - firner is optional Seqner instance of cloned first seen ordinal + firner (Seqner|None): instance of cloned first seen ordinal If cloned mode then firner maybe provided (not None) When firner provided then compare fn of dater and database and first seen if not match then log and add cue notify problem - dater is optional Dater instance of cloned replay datetime + dater (Dater|None): instance of cloned replay datetime If cloned mode then dater maybe provided (not None) When dater provided then use dater for first seen datetime + eager (bool): True means try harder to find validate events by + walking KELs. Enables only being eager + in escrow processing not initial parsing. + False means only use pre-existing information + if any, either percolated attached or in database. + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch ked ilk pre, sn, dig to see how to process pre = serder.pre ked = serder.ked @@ -3134,7 +3779,7 @@ def processEvent(self, serder, sigers, *, wigers=None, "".format(pre, ked)) from ex sn = serder.sn - ilk = serder.ilk # ked["t"] + ilk = serder.ilk # ked["t"] said = serder.said @@ -3153,11 +3798,20 @@ def processEvent(self, serder, sigers, *, wigers=None, firner=firner if self.cloned else None, dater=dater if self.cloned else None, cues=self.cues, - prefixes=self.prefixes, - local=self.local, + eager=eager, + local=local, check=self.check) self.kevers[pre] = kever # not exception so add to kevers + # At this point the inceptive event (icp or dip) given by serder + # together with its attachments has been accepted as valid with finality. + # Events that don't get to here have either been dropped as + # invalid by raising an error or have been escrowed as not + # yet complete enough to decide their validity, i.e. are + # missing signatures, or witness receipts or delegations. + # Any actions that should be triggered as a result of final + # acceptance should follow from here. + if self.direct or self.lax or pre not in self.prefixes: # not own event when owned # create cue for receipt controller or watcher # receipt of actual type is dependent on own type of identifier @@ -3165,22 +3819,16 @@ def processEvent(self, serder, sigers, *, wigers=None, elif not self.direct: # notice of new event self.cues.push(dict(kin="notice", serder=serder)) - if kever.locallyWitnessed(): - # ToDo XXXX need to cue task here kin = "witness" + if self.local and kever.locallyWitnessed(): # + # ToDo XXXX need to cue task here kin = "witness" and process + # cued witness and then combine with reciept above so only + # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if kever.locallyOwned(kever.delegator): # delegator may be None - # ToDo XXXX need to cue task here to approve delegation by generating - # and anchoring SealEvent of serder in delegators KEL - # may include MFA business logic for the delegator i.e. is local - self.cues.push(dict(kin="approveDelegation", - delegator=kever.delegator, - serder=serder)) - else: # not inception so can't verify sigs etc, add to out-of-order escrow self.escrowOOEvent(serder=serder, sigers=sigers, - seqner=delseqner, saider=delsaider, wigers=wigers) + seqner=delseqner, saider=delsaider, wigers=wigers, local=local) raise OutOfOrderError("Out-of-order event={}.".format(ked)) else: # already accepted inception event for pre so already first seen @@ -3206,13 +3854,16 @@ def processEvent(self, serder, sigers, *, wigers=None, if sigers or wigers: # at least one verified sig or wig so log evt # this allows late arriving witness receipts or controller # signatures to be added to the databse - # not first seen inception so ignore return - kever.logEvent(serder, sigers=sigers, wigers=wigers) # idempotent update db logs + # Not first seen version of event so ignore return + # idempotent update db logs + kever.logEvent(serder, sigers=sigers, wigers=wigers) else: # escrow likely duplicitous event self.escrowLDEvent(serder=serder, sigers=sigers) - logger.debug("Likely Duplicitous event Body=%s", serder.pretty()) - raise LikelyDuplicitousError(f"Likely Duplicitous event={serder.said}") + msg = f"Likely Duplicitous Event sn={serder.sn} type={serder.ilk} SAID={serder.said}" + logger.debug(msg) + logger.debug("Duplicitous event body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) else: # rot, drt, or ixn, so sn matters kever = self.kevers[pre] # get existing kever for pre @@ -3221,9 +3872,11 @@ def processEvent(self, serder, sigers, *, wigers=None, if sn > sno: # sn later than sno so out of order escrow # escrow out-of-order event self.escrowOOEvent(serder=serder, sigers=sigers, - seqner=delseqner, saider=delsaider, wigers=wigers) - logger.debug("Out of Order event Body=%s", serder.pretty()) - raise OutOfOrderError(f"Out-of-order event={serder.said}") + seqner=delseqner, saider=delsaider, wigers=wigers, local=local) + msg = f"Out-of-order event sn={serder.sn} type={serder.ilk} SAID={serder.said}" + logger.debug(msg) + logger.debug("Out-of-order event body=\n%s\n", serder.pretty()) + raise OutOfOrderError(msg) elif ((sn == sno) or # inorder event (ixn, rot, drt) or (ilk == Ilks.rot and # superseding recovery rot or @@ -3238,7 +3891,17 @@ def processEvent(self, serder, sigers, *, wigers=None, delseqner=delseqner, delsaider=delsaider, firner=firner if self.cloned else None, dater=dater if self.cloned else None, - check=self.check) + eager=eager, local=local, check=self.check) + + # At this point the non-inceptive event (rot, drt, or ixn) + # given by serder together with its attachments has been + # accepted as valid with finality. + # Events that don't get to here have either been dropped as + # invalid by raising an error or have been escrowed as not + # yet complete enough to decide their validity, i.e. are + # missing signatures, or witness receipts or delegations. + # Any actions that should be triggered as a result of final + # acceptance should follow from here. if self.direct or self.lax or pre not in self.prefixes: # not own event when owned # create cue for receipt controller or watcher @@ -3247,18 +3910,12 @@ def processEvent(self, serder, sigers, *, wigers=None, elif not self.direct: # notice of new event self.cues.push(dict(kin="notice", serder=serder)) - if kever.locallyWitnessed(): - # ToDo XXXX need to cue task here kin = "witness" + if self.local and kever.locallyWitnessed(): # + # ToDo XXXX need to cue task here kin = "witness" and process + # cued witness and then combine with reciept above so only + # one receipt is generated not two self.cues.push(dict(kin="witness", serder=serder)) - if kever.locallyOwned(kever.delegator): # delegator may be None - # ToDo XXXX need to cue task here to approve delegation by generating - # and anchoring SealEvent of serder in delegators KEL - # may include MFA business logic for the delegator i.e. is local - self.cues.push(dict(kin="approveDelegation", - delegator=kever.delegator, - serder=serder)) - else: # maybe duplicitous # check if duplicate of existing valid accepted event @@ -3280,24 +3937,34 @@ def processEvent(self, serder, sigers, *, wigers=None, verfers=werfers) if sigers or wigers: # at least one verified sig or wig so log evt - # not first seen update so ignore return + # this allows late arriving witness receipts or controller + # signatures to be added to the databse + # Not first seen version of event so ignore return + # idempotent update db logs kever.logEvent(serder, sigers=sigers, wigers=wigers) # idempotent update db logs else: # escrow likely duplicitous event self.escrowLDEvent(serder=serder, sigers=sigers) - logger.debug("Likely Duplicitous event Body=%s", serder.pretty()) - raise LikelyDuplicitousError(f"Likely Duplicitous event={serder.said}") + msg = f"Likely Duplicitous Event sn={serder.sn} type={serder.ilk} SAID={serder.said}" + logger.debug(msg) + logger.debug("Duplicitous event body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) + - def processReceiptWitness(self, serder, wigers): + def processReceiptWitness(self, serder, wigers, local=None): """ - Process one witness receipt serder with attached witness sigers + Process one witness receipt serder with attached witness wigers + (indexed signatures) Parameters: - serder is SerderKERI instance of serialized receipt message not receipted event - sigers is list of Siger instances that with witness indexed signatures + serder (SerderKERI): instance of serialized receipt message not receipted event + wigers (list[Siger]): instances that with witness indexed signatures signature in .raw. Index is offset into witness list of latest establishment event for receipted event. Signature uses key pair derived from nontrans witness prefix in associated witness list. + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Receipt dict labels vs # version string @@ -3306,10 +3973,12 @@ def processReceiptWitness(self, serder, wigers): ilk # rct dig # qb64 digest of receipted event """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre dig to process ked = serder.ked pre = serder.pre - sn = serder.sn # Only accept receipt if for last seen version of event at sn @@ -3324,8 +3993,10 @@ def processReceiptWitness(self, serder, wigers): lserder = serdering.SerderKERI(raw=raw) # deserialize event raw if not lserder.compare(said=ked["d"]): # stale receipt at sn discard - raise ValidationError("Stale receipt at sn = {} for rct = {}." - "".format(ked["s"], ked)) + msg = f"Stale receipt at sn = {ked['s']} for rct = {serder.said}." + logger.info(msg) + logger.debug("Stale receipt event body=\n%s\n", serder.pretty()) + raise ValidationError(msg) # process each couple verify sig and write to db wits = [wit.qb64 for wit in self.fetchWitnessState(pre, sn)] @@ -3337,17 +4008,18 @@ def processReceiptWitness(self, serder, wigers): if wiger.verfer.transferable: # skip transferable verfers continue # skip invalid witness prefix - if not self.lax and wiger.verfer.qb64 in self.prefixes: # own is receiptor + if not self.lax and wiger.verfer.qb64 in self.prefixes: # own is witness if pre in self.prefixes: # skip own receiptor of own event # sign own events not receipt them - logger.info("Kevery: skipped own receipt attachment " - "on own event receipt=%s", serder.said) - logger.debug("Receipt Body=\n%s\n", serder.pretty()) + logger.info("Kevery: skipped own receipt attachment" + " on own event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) + continue # skip own receipt attachment on own event - if not self.local: # own receipt on other event when not local - logger.info("Kevery: skipped own receipt attachment " - "on non-local event receipt=%s", serder.said) - logger.debug("Receipt Body=\n%s\n", serder.pretty()) + if not local: # so skip own receipt on other event when non-local source + logger.debug("Kevery: skipped own receipt attachment" + " on nonlocal event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event if wiger.verfer.verify(wiger.raw, lserder.raw): @@ -3357,17 +4029,25 @@ def processReceiptWitness(self, serder, wigers): else: # no events to be receipted yet at that sn so escrow # get digest from receipt message not receipted event self.escrowUWReceipt(serder=serder, wigers=wigers, said=ked["d"]) - raise UnverifiedWitnessReceiptError("Unverified witness receipt={}." - "".format(ked)) + msg = f"Unverified witness receipt={serder.said}" + logger.info(msg) + logger.debug("Event=\n%s\n", serder.pretty()) + raise UnverifiedWitnessReceiptError(msg) - def processReceipt(self, serder, cigars): + def processReceipt(self, serder, cigars, local=None): """ Process one receipt serder with attached cigars + may or may not be a witness receipt. If prefix matches witness then + promote to indexed witness signature and store appropriately. Otherwise + signature is nontrans nonwitness endorser (watcher etc) Parameters: - serder is SerderKERI instance of serialized receipt message not receipted message - cigars is list of Cigar instances that contain receipt couple + serder (SerderKERI): rct instance of serialized receipt message + cigars (list[Cigar]): instances that contain receipt couple signature in .raw and public key in .verfer + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Receipt dict labels vs # version string @@ -3376,6 +4056,9 @@ def processReceipt(self, serder, cigars): ilk # rct dig # qb64 digest of receipted event """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre dig to process ked = serder.ked pre = serder.pre @@ -3405,46 +4088,62 @@ def processReceipt(self, serder, cigars): if not self.lax and cigar.verfer.qb64 in self.prefixes: # own is receiptor if pre in self.prefixes: # skip own receipter of own event # sign own events not receipt them - logger.info("Kevery: skipped own receipt attachment " - "on own event receipt=\n%s\n", serder.said) - logger.debug("Receipt Body=\n%s\n", serder.pretty()) + logger.debug("Kevery process: skipped own receipt attachment" + " on own event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) + continue # skip own receipt attachment on own event - if not self.local: # own receipt on other event when not local - logger.info("Kevery: skipped own receipt attachment " - "on non-local event receipt=%s", serder.said) - logger.debug("Kevery: skipped own receipt attachment " - "on non-local event receipt=\n%s\n", serder.pretty()) + if not local: # skip own receipt on other event when not local + logger.debug("Kevery process: skipped own receipt attachment" + " on nonlocal event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) continue # skip own receipt attachment on non-local event if cigar.verfer.verify(cigar.raw, lserder.raw): wits = [wit.qb64 for wit in self.fetchWitnessState(pre, sn)] rpre = cigar.verfer.qb64 # prefix of receiptor - if rpre in wits: # its a witness receipt + if rpre in wits: # its a witness receipt write in .wigs index = wits.index(rpre) # create witness indexed signature wiger = Siger(raw=cigar.raw, index=index, verfer=cigar.verfer) self.db.addWig(key=dgkey, val=wiger.qb64b) # write to db - else: # write receipt couple to database + else: # not witness rect write receipt couple to database .rcts couple = cigar.verfer.qb64b + cigar.qb64b self.db.addRct(key=dgkey, val=couple) else: # no events to be receipted yet at that sn so escrow self.escrowUReceipt(serder, cigars, said=ked["d"]) # digest in receipt - raise UnverifiedReceiptError("Unverified receipt={}.".format(ked)) + msg = f"Unverified receipt = {serder.said}" + logger.info(msg) + logger.debug("event=\n%s\n", serder.pretty()) + raise UnverifiedReceiptError(msg) + - def processReceiptCouples(self, serder, cigars, firner=None): + def processAttachedReceiptCouples(self, serder, cigars, firner=None, local=None): """ - Process attachment with receipt couple + Process one attachment couple that represents an endorsement from + a nontransferable AID that may or may not be a witness, maybe a watcher. + Originally may have been a non-transferable receipt or key event attachment + if signature is for witness then promote to indexed sig and store + appropriately. + The is the attachment version of .processReceipt Parameters: - serder is SerderKERI instance of receipted serialized event message - to which receipts are attached from replay - cigars is list of Cigar instances that contain receipt couple + serder (SerderKERI): instance serialized event message to which + attachments come from replay (clone) + cigars (list[Cigar]): instances that contain receipt couple signature in .raw and public key in .verfer - firner is Seqner instance of first seen ordinal, + firner (Seqner): instance of first seen ordinal, if provided lookup event by fn = firner.sn + used when in cloned replay mode + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre dig to process ked = serder.ked pre = serder.pre @@ -3453,7 +4152,7 @@ def processReceiptCouples(self, serder, cigars, firner=None): # Only accept receipt if event is latest event at sn. Means its been # first seen and is the most recent first seen with that sn if firner: - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -3476,14 +4175,16 @@ def processReceiptCouples(self, serder, cigars, firner=None): if not self.lax and cigar.verfer.qb64 in self.prefixes: # own is receiptor if pre in self.prefixes: # skip own receipter on own event # sign own events not receipt them - logger.info("Kevery: skipped own receipt attachment" - " on own event receipt=%s", serder.said) - logger.debug("Receipt Body=\n%s\n", serder.pretty()) + logger.debug("Kevery process: skipped own receipt attachment" + " on own event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) + continue # skip own receipt attachment on own event - if not self.local: # own receipt on other event when not local - logger.info("Kevery: skipped own receipt attachment" - " on nonlocal event receipt=\n%s\n", serder.said) - logger.debug("Receipt Body=\n%s\n", serder.pretty()) + if not local: # own receipt on other event when not local + logger.debug("Kevery process: skipped own receipt attachment" + " on nonlocal event receipt=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) + continue # skip own receipt attachment on non-local event if cigar.verfer.verify(cigar.raw, serder.raw): @@ -3498,14 +4199,19 @@ def processReceiptCouples(self, serder, cigars, firner=None): couple = cigar.verfer.qb64b + cigar.qb64b self.db.addRct(key=dgKey(pre, ldig), val=couple) - def processReceiptTrans(self, serder, tsgs): + def processReceiptTrans(self, serder, tsgs, local=None): """ Process one transferable validator receipt (chit) serder with attached sigers + (indexed signatures) who are not controllers. Controllers may only attach signatures to + the associated event not by sending signature attached to receipt of event. Parameters: - serder is chit serder (transferable validator receipt message) - tsgs is tist of tuples from extracted transferable indexed sig groups + serder (serderKERI): rct (transferable validator receipt message) + tsgs (list[tuple]): from extracted transferable indexed sig groups each converted group is tuple of (i,s,d) triple plus list of sigs + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Receipt dict labels vs # version string @@ -3515,6 +4221,9 @@ def processReceiptTrans(self, serder, tsgs): dig # qb64 digest of receipted event """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre, dig,seal to process ked = serder.ked pre = serder.pre @@ -3526,8 +4235,7 @@ def processReceiptTrans(self, serder, tsgs): if ldig is None: # escrow because event does not yet exist in database # take advantage of fact that receipt and event have same pre, sn fields self.escrowTRGroups(serder, tsgs) - logger.debug("Unverified receipt body=\n%s\n", serder.pretty()) - raise UnverifiedTransferableReceiptError("Unverified receipt=%s", serder.said) + raise UnverifiedTransferableReceiptError("Unverified receipt={}.".format(ked)) # retrieve event by dig assumes if ldig is not None that event exists at ldig ldig = bytes(ldig).decode("utf-8") @@ -3535,24 +4243,24 @@ def processReceiptTrans(self, serder, tsgs): lserder = serdering.SerderKERI(raw=bytes(lraw)) # verify digs match if not lserder.compare(said=ldig): # mismatch events problem with replay - raise ValidationError(f"Mismatch receipt of event at sn = {sn} with db.") + raise ValidationError("Mismatch receipt of event at sn = {} with db." + "".format(sn)) for sprefixer, sseqner, saider, sigers in tsgs: # iterate over each tsg if not self.lax and sprefixer.qb64 in self.prefixes: # own is receipter if pre in self.prefixes: # skip own receipter of own event - # sign own events not receipt them - logger.debug("Own pre=%s receiptor of own event \n%s\n", - self.prefixes, serder.pretty()) - raise ValidationError(f"Own pre={self.prefixes} receiptor of own event {serder.said}") - if not self.local: # skip own receipts of nonlocal events - logger.debug("Own pre=%s receiptor of non-local event \n%s\n", serder.prefixes, serder.pretty()) - raise ValidationError(f"Own pre={self.prefixes} receiptor of non-local event {serder.said}") + # sign own events as controller not endorse them via receipt + raise ValidationError("Own pre={} receipter of own event" + " {}.".format(self.prefixes, serder.pretty())) + if not local: # skip own receipts of nonlocal events + raise ValidationError("Own pre={} receipter of nonlocal event " + "{}.".format(self.prefixes, serder.pretty())) # receipted event in db so attempt to get receipter est evt - # retrieve dig of last event at sn of est evt of receipter. + # retrieve dig of last event at sn of est evt of receiptor. sdig = self.db.getKeLast(key=snKey(pre=sprefixer.qb64b, sn=sseqner.sn)) if sdig is None: - # receipter's est event not yet in receipters's KEL + # receiptor's est event not yet in receiptors's KEL # so need cue to discover est evt KEL for receipter from watcher etc self.escrowTReceipts(serder, sprefixer, sseqner, saider, sigers) raise UnverifiedTransferableReceiptError("Unverified receipt: " @@ -3560,7 +4268,7 @@ def processReceiptTrans(self, serder, tsgs): "receipter for event={}." "".format(ked)) - # retrieve last event itself of receipter est evt from sdig. + # retrieve last event itself of receiptor est evt from sdig. sraw = self.db.getEvt(key=dgKey(pre=sprefixer.qb64b, dig=bytes(sdig))) # assumes db ensures that sraw must not be none because sdig was in KE sserder = serdering.SerderKERI(raw=bytes(sraw)) @@ -3576,7 +4284,7 @@ def processReceiptTrans(self, serder, tsgs): " dig = {} from pre ={}, no keys." "".format(saider.qb64, sprefixer.qb64)) - for siger in sigers: + for siger in sigers: # endorser (non-controller) signatures if siger.index >= len(sverfers): raise ValidationError("Index = {} to large for keys." "".format(siger.index)) @@ -3587,16 +4295,24 @@ def processReceiptTrans(self, serder, tsgs): self.db.addVrc(key=dgKey(pre=pre, dig=ldig), val=quadruple) # dups kept - def processReceiptQuadruples(self, serder, trqs, firner=None): + def processAttachedReceiptQuadruples(self, serder, trqs, firner=None, local=None): """ - Process one attachment quadruple that comprises a transferable receipt + Process one attachment quadruple that represents an endorsement from + a transferable AID that is not the controller. Maybe a watcher. + Originally may have been a transferable receipt or key event attachment + + This is the attachement version of .processReceiptTrans Parameters: - serder is chit serder (transferable validator receipt message) - trqs is list of tuples (quadruples) of form - (prefixer, seqner, diger, siger) - firner is Seqner instance of first seen ordinal, + serder (serderKERI): instance serialized event message to which + attachments come from replay (clone) + trqs (list[tuple]): quadruples of (prefixer, seqner, diger, siger) + firner (Seqner): instance of first seen ordinal, if provided lookup event by fn = firner.sn + used when in cloned replay mode + local (bool|None): True means local (protected) event source. + False means remote (unprotected). + None means use default .local . Seal labels i pre # qb64 prefix of receipter @@ -3604,13 +4320,16 @@ def processReceiptQuadruples(self, serder, trqs, firner=None): d dig # qb64 digest of est event for receipter keys """ + local = local if local is not None else self.local + local = True if local else False # force boolean + # fetch pre, dig,seal to process ked = serder.ked pre = serder.pre sn = serder.sn if firner: # retrieve last event by fn ordinal - ldig = self.db.getFe(key=fnKey(pre=pre, sn=firner.sn)) + ldig = self.db.getFe(key=fnKey(pre=pre, fn=firner.sn)) else: # Only accept receipt if for last seen version of receipted event at sn ldig = self.db.getKeLast(key=snKey(pre=pre, sn=sn)) # retrieve dig of last event at sn. @@ -3621,7 +4340,7 @@ def processReceiptQuadruples(self, serder, trqs, firner=None): raise ValidationError("Own pre={} replay attached transferable " "receipt quadruple of own event {}." "".format(self.prefixes, serder.pretty())) - if not self.local: # skip own trans receipt quadruples of nonlocal events + if not local: # skip own trans receipt quadruples of nonlocal events raise ValidationError("Own pre={} seal in replay attached " "transferable receipt quadruples of nonlocal" " event {}.".format(self.prefixes, serder.pretty())) @@ -3670,9 +4389,8 @@ def processReceiptQuadruples(self, serder, trqs, firner=None): siger.verfer = sverfers[siger.index] # assign verfer if not siger.verfer.verify(siger.raw, serder.raw): # verify sig - msg = (f"Kevery unescrow error: Bad escrowed trans receipt sig at " - f"pre={pre} sn={sn:x} receiptor={sprefixer.qb64}") - logger.trace(msg) + msg = f"Bad escrowed trans receipt sig pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # good sig so write receipt quadruple to database @@ -3684,11 +4402,10 @@ def processReceiptQuadruples(self, serder, trqs, firner=None): else: # escrow either receiptor or receipted event not yet in database self.escrowTRQuadruple(serder, sprefixer, sseqner, saider, siger) - msg = (f"Unverified receipt: " - f"missing associated event for transferable " - f"validator receipt quadruple for event={serder.said}") + msg = (f"Unverified receipt: missing associated event for transferable validator" + f"receipt quadruple for event {serder.said}") logger.info(msg) - logger.debug("Event Body=\n%s\n", serder.pretty()) + logger.debug("Event=\n%s\n", serder.pretty()) raise UnverifiedTransferableReceiptError(msg) def removeStaleReplyEndRole(self, saider): @@ -3704,7 +4421,6 @@ def removeStaleReplyLocScheme(self, saider): """ pass - def registerReplyRoutes(self, router): """ Register the routes for processing messages embedded in `rpy` event messages @@ -3715,10 +4431,9 @@ def registerReplyRoutes(self, router): router.addRoute("/end/role/{action}", self, suffix="EndRole") router.addRoute("/loc/scheme", self, suffix="LocScheme") router.addRoute("/ksn/{aid}", self, suffix="KeyStateNotice") + router.addRoute("/watcher/{aid}/{action}", self, suffix="AddWatched") - - def processReplyEndRole(self, *, serder, saider, route, - cigars=None, tsgs=None, **kwargs): + def processReplyEndRole(self, *, serder, saider, route, cigars=None, tsgs=None, **kwargs): """ Process one reply message for route = /end/role/add or /end/role/cut with either attached nontrans receipt couples in cigars or attached trans @@ -3806,7 +4521,10 @@ def processReplyEndRole(self, *, serder, saider, route, aid=aid, osaider=osaider, cigars=cigars, tsgs=tsgs) if not accepted: - raise UnverifiedReplyError(f"Unverified end role reply. {serder.ked}") + msg = f"Unverified end role reply = {serder.said} role = {role}" + logger.debug(f"Kevery: %s", msg) + logger.debug(f"Event=\n%s\n", serder.pretty()) + raise UnverifiedReplyError(msg) self.updateEnd(keys=keys, saider=saider, allowed=allowed) # update .eans and .ends @@ -3903,8 +4621,10 @@ def processReplyLocScheme(self, *, serder, saider, route, aid=aid, osaider=osaider, cigars=cigars, tsgs=tsgs) if not accepted: - logger.debug("Unverified loc scheme reply body=\n%s\n", serder.pretty()) - raise UnverifiedReplyError(f"Unverified loc scheme reply for URL {serder.ked['a']['url']}") + msg = f"Unverified loc scheme reply URL={url} SAID={serder.said}" + logger.debug(msg) + logger.debug("Event Body=\n%s\n", serder.pretty()) + raise UnverifiedReplyError(msg) self.updateLoc(keys=keys, saider=saider, url=url) # update .lans and .locs @@ -4037,7 +4757,7 @@ def processReplyKeyStateNotice(self, *, serder, saider, route, ksaider = coring.Saider(qb64=diger.qb64) self.updateKeyState(aid=aid, ksr=ksr, saider=ksaider, dater=dater) - self.cues.push(dict(kin="keyStateSaved", ksn=ksr._asdict())) + self.cues.push(dict(kin="keyStateSaved", ksn=asdict(ksr))) def updateEnd(self, keys, saider, allowed=None): """ @@ -4101,6 +4821,100 @@ def removeKeyState(self, saider): self.db.ksns.rem(keys=keys) self.db.kdts.rem(keys=keys) + def processReplyAddWatched(self, *, serder, saider, route, + cigars=None, tsgs=None, **kwargs): + """ Process one reply message for adding an AID for a watcher to watch + + Process one reply message for adding an AID for a watcher to watch = /watcher/{aid}/add + with either attached nontrans receipt couples in cigars or attached trans + indexed sig groups in tsgs. + Assumes already validated saider, dater, and route from serder.ked + + Parameters: + serder (SerderKERI): instance of reply msg (SAD) + saider (Saider): instance from said in serder (SAD) + route (str): reply route + cigars (list): of Cigar instances that contain nontrans signing couple + signature in .raw and public key in .verfer + tsgs (list): tuples (quadruples) of form + (prefixer, seqner, diger, [sigers]) where: + prefixer is pre of trans endorser + seqner is sequence number of trans endorser's est evt for keys for sigs + diger is digest of trans endorser's est evt for keys for sigs + [sigers] is list of indexed sigs from trans endorser's keys from est evt + + Reply Message: + { + "v" : "KERI10JSON00011c_", + "t" : "rpy", + "d": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "dt": "2020-08-22T17:50:12.988921+00:00", + "r" : "/watcher/BrHLayDN-mXKv62DAjFLX1_Y5yEUe0vA9YPe_ihiKYHE/add", + "a" : + { + "cid": "EyX-zd8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM" + "oid": "EM0-i05TNZJZAoH3UR2nmLaU6JwyvPzhzS6YAfSVbMC5" + "oobi": "http://example.com/oobi/EyX-zd8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM" + } + } + + """ + aid = kwargs["aid"] + action = kwargs["action"] + # reply specific logic + if not route.startswith("/watcher"): + raise ValidationError(f"Usupported route={route} in {Ilks.rpy} " + f"msg={serder.ked}.") + + # reply specific logic + if action == "add": + enabled = True + elif action == "cut": + enabled = False + else: # unsupported route + raise ValidationError(f"Usupported route={route} in {Ilks.rpy} " + f"msg={serder.ked}.") + route = f"/watcher/{aid}" # escrow based on route base + cigars = cigars if cigars is not None else [] + tsgs = tsgs if tsgs is not None else [] + + data = serder.ked["a"] + cid = data["cid"] + oid = data["oid"] + oobi = data["oobi"] if "oobi" in data else None + + keys = (cid, aid, oid) + + osaider = self.db.wwas.get(keys=keys) # get old said if any + + # BADA Logic + accepted = self.rvy.acceptReply(serder=serder, saider=saider, route=route, + aid=cid, osaider=osaider, cigars=cigars, + tsgs=tsgs) + if not accepted: + raise UnverifiedReplyError(f"Unverified watcher add reply. {serder.ked}") + + if oobi: + self.db.oobis.pin(keys=(oobi,), val=OobiRecord(date=help.nowIso8601())) + self.updateWatched(keys=keys, saider=saider, enabled=enabled) + + def updateWatched(self, keys, saider, enabled=None): + """ + Update loc auth database .lans and loc database .locs. + + Parameters: + keys (tuple): of key strs for databases (eid, scheme) + saider (Saider): instance from said in reply serder (SAD) + enabled (bool): True means add observed to watcher, False means remove (cut) + """ + self.db.wwas.pin(keys=keys, val=saider) # overwrite + if observed := self.db.obvs.get(keys=keys): # preexisting record + observed.enabled = enabled # update preexisting record + else: # no preexisting record + observed = basing.ObservedRecord(enabled=enabled, datetime=helping.nowIso8601()) # create new record + + self.db.obvs.pin(keys=keys, val=observed) # overwrite + def processQuery(self, serder, source=None, sigers=None, cigars=None): """ Process query mode replay message for collective or single element query. @@ -4122,41 +4936,53 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): # do signature validation and replay attack prevention logic here # src, dt, route + if source is None and cigars: + dest = cigars[0].verfer.qb64 + else: + dest = source.qb64 + if route == "logs": pre = qry["i"] src = qry["src"] anchor = qry["a"] if "a" in qry else None sn = int(qry["s"], 16) if "s" in qry else None + fn = int(qry["fn"], 16) if "fn" in qry else 0 if pre not in self.kevers: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - logger.debug("Query not found error. Query Body=%s", ked) - raise QueryNotFoundError(f"Query not found error for at route {ked['r']} for evt {ked['d']}") + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) kever = self.kevers[pre] if anchor: - if not self.db.findAnchoringSealEvent(pre=pre, seal=anchor): + if not self.db.fetchAllSealingEventByEventSeal(pre=pre, seal=anchor): self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - logger.debug("Query not found error. Query Body=%s", ked) - raise QueryNotFoundError(f"Query not found error at route {ked['r']} for evt {ked['d']}") + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) elif sn is not None: if kever.sner.num < sn or not self.db.fullyWitnessed(kever.serder): self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - logger.debug("Query not found error. Query Body=%s", ked) - raise QueryNotFoundError(f"Query not found error at route {ked['r']} for evt {ked['d']}") + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) msgs = list() # outgoing messages - for msg in self.db.clonePreIter(pre=pre, fn=0): + for msg in self.db.clonePreIter(pre=pre, fn=fn): msgs.append(msg) - if kever.delegator: - cloner = self.db.clonePreIter(pre=kever.delegator, fn=0) # create iterator at 0 + if kever.delpre: + cloner = self.db.clonePreIter(pre=kever.delpre, fn=0) # create iterator at 0 for msg in cloner: msgs.append(msg) if msgs: - self.cues.push(dict(kin="replay", src=src, msgs=msgs, dest=source.qb64)) + self.cues.push(dict(kin="replay", pre=pre, src=src, msgs=msgs, dest=dest)) elif route == "ksn": pre = qry["i"] @@ -4164,8 +4990,10 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): if pre not in self.kevers: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - logger.debug("Query not found error. Query Body=%s", ked) - raise QueryNotFoundError(f"Query not found error at route {ked['r']} for evt {ked['d']}") + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) kever = self.kevers[pre] @@ -4175,12 +5003,14 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): if len(wigers) < kever.toader.num: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - logger.debug("Query not found error. Query Body=%s", ked) - raise QueryNotFoundError(f"Query not found error at route {ked['r']} for evt {ked['d']}") + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) rserder = reply(route=f"/ksn/{src}", data=kever.state()._asdict()) self.cues.push(dict(kin="reply", src=src, route="/ksn", serder=rserder, - dest=source.qb64)) + dest=dest)) elif route == "mbx": pre = qry["i"] @@ -4189,8 +5019,10 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): if pre not in self.kevers: self.escrowQueryNotFoundEvent(serder=serder, prefixer=source, sigers=sigers, cigars=cigars) - logger.debug("Query not found error. Query Body=%s", ked) - raise QueryNotFoundError(f"Query not found error at route {ked['r']} for evt {ked['d']}") + msg = f"Query not found error on event route={route} SAID={serder.said}" + logger.debug(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise QueryNotFoundError(msg) self.cues.push(dict(kin="stream", serder=serder, pre=pre, src=src, topics=topics)) # if pre in self.kevers: @@ -4199,8 +5031,10 @@ def processQuery(self, serder, source=None, sigers=None, cigars=None): # self.cues.push(dict(kin="stream", serder=serder, pre=pre, src=src, topics=topics)) else: self.cues.push(dict(kin="invalid", serder=serder)) - logger.debug("Invalid Query error. Query Body=%s", ked) - raise ValidationError(f"invalid query message {ilk} at route {ked['r']} for evt = {ked['d']}") + msg = f"Invalid query message {ilk} for event route={route} SAID={serder.said}" + logger.info(msg) + logger.debug("Query Body=\n%s\n", serder.pretty()) + raise ValidationError(msg) def fetchEstEvent(self, pre, sn): """ @@ -4215,15 +5049,17 @@ def fetchEstEvent(self, pre, sn): found = False while not found: - dig = bytes(self.db.getKeLast(key=snKey(pre, sn))) + dig = self.db.getKeLast(key=snKey(pre, sn)) if not dig: return None # retrieve event by dig - raw = bytes(self.db.getEvt(key=dgKey(pre=pre, dig=dig))) + dig = bytes(dig) + raw = self.db.getEvt(key=dgKey(pre=pre, dig=dig)) if not raw: return None + raw = bytes(raw) serder = serdering.SerderKERI(raw=raw) # deserialize event raw if serder.ked["t"] in (Ilks.icp, Ilks.dip, Ilks.rot, Ilks.drt): return serder # establishment event so return @@ -4232,7 +5068,49 @@ def fetchEstEvent(self, pre, sn): if sn < 0: # no more events return None - def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None): + + def escrowMFEvent(self, serder, sigers, wigers=None, + seqner=None, saider=None, local=True): + """ + Update associated logs for escrow of MisFit event + + Parameters: + serder (SerderKERI): instance of event + sigers (list): of Siger instance for event + seqner (Seqner): instance of sn of event delegatint/issuing event if any + saider (Saider): instance of dig of event delegatint/issuing event if any + wigers (list): of witness signatures + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote + """ + local = True if local else False + dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) + self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) + self.db.putEvt(dgkey, serder.raw) + if wigers: + self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) + if seqner and saider: + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + self.db.misfits.add(keys=(serder.pre, serder.snh), val=serder.saidb) + # log escrowed + logger.debug("Kevery process: escrowed misfit event=\n%s", serder.pretty()) + + + def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None, local=True): """ Update associated logs for escrow of Out-of-Order event @@ -4242,20 +5120,34 @@ def escrowOOEvent(self, serder, sigers, seqner=None, saider=None, wigers=None): seqner (Seqner): instance of sn of event delegatint/issuing event if any saider (Saider): instance of dig of event delegatint/issuing event if any wigers (list): of witness signatures + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) self.db.putEvt(dgkey, serder.raw) - self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) if wigers: self.db.putWigs(dgkey, [siger.qb64b for siger in wigers]) if seqner and saider: - couple = seqner.qb64b + saider.qb64b - self.db.putPde(dgkey, couple) # idempotent + #couple = seqner.qb64b + saider.qb64b + #self.db.putUde(dgkey, couple) # idempotent + self.db.udes.put(keys=dgkey, val=(seqner, saider)) # idempotent + self.db.addOoe(snKey(serder.preb, serder.sn), serder.saidb) # log escrowed - logger.info("Kevery: escrowed out of order event=\n%s\n", serder.said) - logger.debug("Event Body=\n%s\n", serder.pretty()) + logger.debug("Kevery process: escrowed out of order event=\n%s", serder.pretty()) def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): """ @@ -4272,7 +5164,7 @@ def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) self.db.putEvt(dgkey, serder.raw) - self.db.addQnf(dgkey, serder.saidb) + self.db.qnfs.add(keys=(prefixer.qb64, serder.said), val=serder.saidb) for cigar in cigars: self.db.addRct(key=dgkey, val=cigar.verfer.qb64b + cigar.qb64b) @@ -4281,22 +5173,36 @@ def escrowQueryNotFoundEvent(self, prefixer, serder, sigers, cigars=None): logger.trace("Kevery: escrowed query not found event = %s", serder.said) logger.trace("Event Body=\n%s\n", serder.pretty()) - def escrowLDEvent(self, serder, sigers): + def escrowLDEvent(self, serder, sigers, local=True): """ Update associated logs for escrow of Likely Duplicitous event Parameters: serder is SerderKERI instance of event sigers is list of Siger instance for event + local (bool): event source for validation logic + True means event source is local (protected). + False means event source is remote (unprotected). + Event validation logic is a function of local or remote """ + local = True if local else False dgkey = dgKey(serder.preb, serder.saidb) + if esr := self.db.esrs.get(keys=dgkey): # preexisting esr + if local and not esr.local: # local overwrites prexisting remote + esr.local = local + self.db.esrs.pin(keys=dgkey, val=esr) + # otherwise don't change + else: # not preexisting so put + esr = basing.EventSourceRecord(local=local) + self.db.esrs.put(keys=dgkey, val=esr) + self.db.putDts(dgkey, helping.nowIso8601().encode("utf-8")) self.db.putSigs(dgkey, [siger.qb64b for siger in sigers]) self.db.putEvt(dgkey, serder.raw) self.db.addLde(snKey(serder.preb, serder.sn), serder.saidb) # log duplicitous - logger.info("Kevery process: escrowed likely duplicitous event=\n%s\n", - json.dumps(serder.ked, indent=1)) + logger.debug("Kevery process: escrowed likely duplicitous event=\n%s", + json.dumps(serder.ked, indent=1)) def escrowUWReceipt(self, serder, wigers, said): """ @@ -4331,8 +5237,8 @@ def escrowUWReceipt(self, serder, wigers, said): couple = said.encode("utf-8") + wiger.qb64b self.db.addUwe(key=snKey(serder.preb, serder.sn), val=couple) # log escrowed - logger.info("Kevery process: escrowed unverified witness indexed receipt" - " of pre= %s sn=%x dig=%s\n", serder.pre, serder.sn, said) + logger.debug("Kevery process: escrowed unverified witness indexed receipt" + " of pre= %s sn=%x dig=%s", serder.pre, serder.sn, said) def escrowUReceipt(self, serder, cigars, said): """ @@ -4362,8 +5268,8 @@ def escrowUReceipt(self, serder, cigars, said): triple = said.encode("utf-8") + cigar.verfer.qb64b + cigar.qb64b self.db.addUre(key=snKey(serder.preb, serder.sn), val=triple) # should be snKey # log escrowed - logger.info("Kevery process: escrowed unverified receipt of pre= %s " - " sn=%x dig=%s\n", serder.pre, serder.sn, said) + logger.debug("Kevery process: escrowed unverified receipt of pre= %s " + " sn=%x dig=%s", serder.pre, serder.sn, said) def escrowTRGroups(self, serder, tsgs): """ @@ -4406,9 +5312,9 @@ def escrowTRGroups(self, serder, tsgs): quintuple = prelet + siger.qb64b # quintuple self.db.addVre(key=snKey(serder.preb, serder.sn), val=quintuple) # log escrowed - logger.info("Kevery process: escrowed unverified transferable receipt " - "of pre=%s sn=%x dig=%s by pre=%s\n", serder.pre, - serder.sn, serder.ked["d"], prefixer.qb64) + logger.debug("Kevery process: escrowed unverified transferable receipt " + "of pre=%s sn=%x dig=%s by pre=%s", serder.pre, + serder.sn, serder.ked["d"], prefixer.qb64) def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): """ @@ -4420,7 +5326,7 @@ def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): prefixer is Prefixer instance of prefix of receipter seqner is Seqner instance of sn of est event of receiptor saider is Saider instance of said of est event of receiptor - igers is list of Siger instances of multi-sig of receiptor + sigers is list of Siger instances of multi-sig of receiptor escrow quintuple for each siger quintuple = edig+pre+snu+dig+sig @@ -4428,7 +5334,7 @@ def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): edig is receipted event dig (serder.dig) pre is receipter prefix snu is receipter est event sn - dig is receipt est evant dig + dig is receipt est event dig sig is indexed sig of receiptor of receipted event """ # Receipt dig algo may not match database dig. So must always @@ -4448,8 +5354,8 @@ def escrowTReceipts(self, serder, prefixer, seqner, saider, sigers): quintuple = prelet + siger.qb64b # quintuple self.db.addVre(key=snKey(serder.preb, serder.sn), val=quintuple) # log escrowed - logger.info("Kevery process: escrowed unverified transferable receipt " - "of pre=%s sn=%x dig=%s by pre=%s\n", serder.pre, + logger.debug("Kevery process: escrowed unverified transferable receipt " + "of pre=%s sn=%x dig=%s by pre=%s", serder.pre, serder.sn, serder.ked["d"], prefixer.qb64) def escrowTRQuadruple(self, serder, sprefixer, sseqner, saider, siger): @@ -4481,25 +5387,28 @@ def escrowTRQuadruple(self, serder, sprefixer, sseqner, saider, siger): saider.qb64b + siger.qb64b) self.db.addVre(key=snKey(serder.preb, serder.sn), val=quintuple) # log escrowed - logger.info("Kevery process: escrowed unverified transferabe validator " - "receipt of pre= %s sn=%x dig=%s\n", serder.pre, serder.sn, - serder.said) + logger.debug("Kevery process: escrowed unverified transferabe validator " + "receipt of pre= %s sn=%x dig=%s", serder.pre, serder.sn, + serder.said) def processEscrows(self): """ - Iterate through escrows and process any that may now be finalized + Iterate throush escrows and process any that may now be finalized Parameters: """ + try: self.processEscrowOutOfOrders() self.processEscrowUnverWitness() self.processEscrowUnverNonTrans() self.processEscrowUnverTrans() + self.processEscrowPartialDels() self.processEscrowPartialWigs() self.processEscrowPartialSigs() self.processEscrowDuplicitous() self.processQueryNotFound() + except Exception as ex: # log diagnostics errors etc if logger.isEnabledFor(logging.DEBUG): logger.trace("Kevery: other escrow process error: %s\n", ex.args[0]) @@ -4546,15 +5455,22 @@ def processEscrowOutOfOrders(self): key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getOoeItemsNextIter(key=key): + for ekey, edig in self.db.getOoeItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local source so raise ValidationError which unescrows below + msg = f"OOO Missing escrowed event source at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) + dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"Kevery ooo escrow unescrow error: Missing event datetime at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"OOO Missing escrowed event datetime at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes @@ -4562,16 +5478,16 @@ def processEscrowOutOfOrders(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutOOE): # escrow stale so raise ValidationError which unescrows below - msg = f"Kevery ooo escrow unescrow error: Stale event escrow at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"OOO Stale event escrow at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so raise ValidationError which unescrows below - msg = f"Kevery ooo escrow unescrow error: Missing event at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"OOO Missing escrowed event at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -4580,8 +5496,8 @@ def processEscrowOutOfOrders(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - msg = f"Kevery ooo escrow unescrow error: Missing event sigs at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"OOO Missing escrowed event sigs at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # process event @@ -4591,7 +5507,7 @@ def processEscrowOutOfOrders(self): wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers) + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, local=esr.local) # If process does NOT validate event with sigs, becasue it is # still out of order then process will attempt to re-escrow @@ -4610,29 +5526,30 @@ def processEscrowOutOfOrders(self): except OutOfOrderError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery ooo escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery ooo escrow unescrow failed: %s\n", ex.args[0]) + logger.trace("Kevery OOO escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery OOO escrow unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delOoe(snKey(pre, sn), edig) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.debug("Kevery: ooo escrow other error on escrow: %s\n", ex.args[0]) - logger.exception("Kevery: ooo escrow other error on : %s\n", ex.args[0]) + logger.debug("Kevery: OOO escrow other error on escrow: %s\n", ex.args[0]) + logger.exception("Kevery: OOO escrow other error on : %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delOoe(snKey(pre, sn), edig) # removes one escrow at key val - logger.info("Kevery ooo escrow unescrow succeeded in valid event: " - "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) - logger.debug("Event Body=\n%s\n", eserder.pretty()) + logger.info("Kevery out of order unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug("Event=\n%s\n", eserder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break key = ekey # setup next while iteration, with key after ekey + def processEscrowPartialSigs(self): """ Process events escrowed by Kever that were only partially fulfilled, @@ -4668,120 +5585,140 @@ def processEscrowPartialSigs(self): If successful then remove from escrow table """ - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPseItemsNextIter(key=key): - eserder = None - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - dgkey = dgKey(pre, bytes(edig)) - # check date if expired then remove escrow. - dtb = self.db.getDts(dgkey) - if dtb is None: # otherwise is a datetime as bytes - # no date time so raise ValidationError which unescrows below] - msg = f"Kevery: partial sig escrow unescrow error: Missing event datetime at dig = {bytes(edig)}" - logger.trace(msg) - raise ValidationError(msg) - - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): - # escrow stale so raise ValidationError which unescrows below - msg = f"Kevery: partial sig escrow unescrow error: Stale event escrow at dig = {bytes(edig)}", - logger.trace(msg) - raise ValidationError(msg) - - # get the escrowed event using edig - eraw = self.db.getEvt(dgkey) - if eraw is None: - # no event so raise ValidationError which unescrows below - msg = f"Kevery: partial sig escrow unescrow error: Missing event at dig = {bytes(edig)}" - logger.trace(msg) - raise ValidationError(msg) - - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - # get sigs and attach - sigs = self.db.getSigs(dgkey) - if not sigs: # otherwise its a list of sigs - # no sigs so raise ValidationError which unescrows below - msg = f"Kevery: partial sig escrow unescrow error: Missing event sigs at dig = {bytes(edig)}" - logger.trace(msg) - raise ValidationError(msg) - - # seal source (delegator issuer if any) - delseqner = delsaider = None - couple = self.db.getPde(dgkey) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) - elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - if eserder.pre in self.kevers: - delpre = self.kevers[eserder.pre].delegator - else: - delpre = eserder.ked["di"] - - seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) - srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) - if srdr is not None: - delseqner = coring.Seqner(sn=srdr.sn) - delsaider = coring.Saider(qb64=srdr.said) - couple = delseqner.qb64b + delsaider.qb64b - self.db.putPde(dgkey, couple) - - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - self.processEvent(serder=eserder, sigers=sigers, - delseqner=delseqner, delsaider=delsaider) - - # If process does NOT validate sigs or delegation seal (when delegated), - # but there is still one valid signature then process will - # attempt to re-escrow and then raise MissingSignatureError - # or MissingDelegationSealError (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pse escrow is called by - # Kever.self.escrowPSEvent Which calls - # self.db.addPse(snKey(pre, sn), serder.digb) - # which in turn will not enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pse db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. - - except (MissingSignatureError, MissingDelegationError) as ex: - # still waiting on missing sigs or missing seal to validate - if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery: partial sig escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery: partial sig escrow unescrow failed: %s\n", ex.args[0]) - - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) - - if logger.isEnabledFor(logging.DEBUG): - logger.trace("Kevery: partial sig escrow other error on unescrow: %s\n", ex.args[0]) - logger.exception("Kevery: partial sig escrow other error on unescrow: %s\n", ex.args[0]) + #key = ekey = b'' # both start same. when not same means escrows found + #while True: # break when done + for ekey, edig in self.db.getPseItemIter(key=b''): + eserder = None + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local source so raise ValidationError which unescrows below + msg = f"PSE Missing escrowed event source at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val - self.db.delPde(dgkey) # remove escrow if any + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + msg = f"PSE Missing escrowed event datetime at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): - self.cues.push(dict(kin="psUnescrow", serder=eserder)) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): + # escrow stale so raise ValidationError which unescrows below + msg = f"PSE Stale event escrow at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - logger.trace("Kevery: partial sig escrow unescrow succeeded in valid event: " - "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) - logger.trace("Event Body=\n%s\n", json.dumps(eserder.ked, indent=1)) + # get the escrowed event using edig + eraw = self.db.getEvt(dgkey) + if eraw is None: + # no event so so raise ValidationError which unescrows below + msg = f"PSE Missing escrowed evt at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + # get sigs and attach + sigs = self.db.getSigs(dgkey) + if not sigs: # otherwise its a list of sigs + # no sigs so raise ValidationError which unescrows below + msg = f"PSE Missing escrowed evt sigs at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + if not wigs: # empty list wigs witness sigs not wits + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: No event wigs yet at." + "dig = %s", bytes(edig)) + + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=dgkey)): + delseqner, delsaider = couple + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) + + # If process does NOT validate sigs or delegation seal (when delegated), + # but there is still one valid signature then process will + # attempt to re-escrow and then raise MissingSignatureError + # or MissingDelegationSealError (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pse escrow is called by + # Kever.self.escrowPSEvent Which calls + # self.db.addPse(snKey(pre, sn), serder.digb) + # which in turn will not enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pse db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + + except MissingSignatureError as ex: # MissingDelegationError) + # still waiting on missing sigs or missing seal to validate + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: PSE unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: PSE unescrow failed: %s\n", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs so remove from escrow + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow + + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) + + if logger.isEnabledFor(logging.DEBUG): + logger.trace("Kevery: PSE other error on unescrow: %s\n", + ex.args[0]) + logger.exception("Kevery: PSE other error on unescrow: %s\n", + ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPse(snKey(pre, sn), edig) # removes one escrow at key val + self.db.udes.rem(keys=dgkey) # remove escrow if any + + if eserder is not None and eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + self.cues.push(dict(kin="psUnescrow", serder=eserder)) + + logger.trace("Kevery: partial sig escrow unescrow succeeded in valid event: " + "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) + logger.debug("event=\n%s\n", eserder.pretty()) + + #if ekey == key: # still same so no escrows found on last while iteration + #break + #key = ekey # setup next while iteration, with key after ekey def processEscrowPartialWigs(self): """ @@ -4819,116 +5756,297 @@ def processEscrowPartialWigs(self): Process event as if it came in over the wire If successful then remove from escrow table """ + for ekey, edig in self.db.getPweItemIter(key=b''): + try: + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local source so raise ValidationError which unescrows below + msg = f"PWE Missing escrowed event source at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - key = ekey = b'' # both start same. when not same means escrows found - while True: # break when done - for ekey, edig in self.db.getPweItemsNextIter(key=key): - try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item - # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) - if dtb is None: # othewise is a datetime as bytes - # no date time so raise ValidationError which unescrows below - msg = f"Kevery: partial wig escrow unescrow error: Missing event datetime at dig = {bytes(edig)}" - logger.trace(msg) - raise ValidationError(msg) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + msg = f"PWE Missing escrowed event datetime at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - # do date math here and discard if stale nowIso8601() bytes - dtnow = helping.nowUTC() - dte = helping.fromIso8601(bytes(dtb)) - if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): - # escrow stale so raise ValidationError which unescrows below - msg = f"Kevery: partial wig escrow unescrow error: Stale event escrow at dig = {bytes(edig)}" - logger.trace(msg) - raise ValidationError(msg) + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + msg = f"PWE Stale event escrow at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) - if eraw is None: - # no event so so raise ValidationError which unescrows below - msg = f"Kevery: partial wig escrow unescrow error: Missing event at dig = {bytes(edig)}" - logger.trace(msg) - raise ValidationError(msg) + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so so raise ValidationError which unescrows below + msg = f"PWE Missing escrowed evt at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - # get sigs - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs - if not sigs: # empty list - # no sigs so raise ValidationError which unescrows below - msg = f"Kevery: partial wig escrow unescrow error: Missing event sigs at dig = {bytes(edig)}" - logger.trace(msg) - raise ValidationError(msg) + # get sigs + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + msg = f"PWE Missing escrowed evt sigs at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - # get wigs - wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + # get witness signatures (wigs not wits) + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + + if not wigs: # empty list + # wigs maybe empty if not wits or if wits while waiting + # for first witness signature + # which may not arrive until some time after event is fully signed + # so just log for debugging but do not unescrow by raising + # ValidationError + logger.debug("Kevery unescrow wigs: PW No event wigs yet at." + "dig = %s", bytes(edig).decode()) + + # raise ValidationError("Missing escrowed evt wigs at " + # "dig = {}.".format(bytes(edig))) + + # process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # seal source (delegator issuer if any) + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(pre, bytes(edig)))): + delseqner, delsaider = couple + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + ## idempotent + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) + + # If process does NOT validate wigs then process will attempt + # to re-escrow and then raise MissingWitnessSignatureError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pwe escrow is called by + # Kever.self.escrowPWEvent Which calls + # self.db.addPwe(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrowed pwe db. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except MissingWitnessSignatureError as ex: # MissingDelegationError + # still waiting on missing witness sigs or delegation + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: PWE unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: PWE unescrow failed: %s\n", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on wigs so remove from escrow + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + #self.db.udes.rem(keys=dgkey) # leave here since could PartialDelegationEscrow + if logger.isEnabledFor(logging.TRACE): + logger.trace("Kevery: PWE other error on unescrow: %s\n", + ex.args[0]) + logger.exception("Kevery: PWE other error unescrow: %s\n", + ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val + self.db.udes.rem(keys=dgkey) # remove escrow if any + logger.info("Kevery: partial wig escrow unescrow succeeded in valid event: " + "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) + logger.debug("Event=\n%s\n", eserder.pretty()) + + + def processEscrowPartialDels(self): + """ + Process delgated events escrowed by Kever that were only partially fulfilled + due to missing or unverified delegation seals from delegators. + Events only make into this escrow after fully signed and if witnessed + fully witnessed. - if not wigs: # empty list - # wigs maybe empty while waiting for first witness signature - # which may not arrive until some time after event is fully signed - # so just log for debugging but do not unescrow by raising - # ValidationError - logger.trace("Kevery unescrow wigs: No event wigs yet at." - "dig = %s\n", bytes(edig)) + db.pdes is an instance of subing.IoSetSuber and uses instance methods + for access to the underlying database. + Escrowed items in .pdes are indexed in database table keyed by prefix and + sequence number with multiple entries at same key held in insertion order. + This allows FIFO processing of escrowed events with same prefix and sn. - # raise ValidationError("Missing escrowed evt wigs at " - # "dig = {}.".format(bytes(edig))) + Value in each .pdes entry is dgkey (SAID) of event stored in db.evts where + db.evts holds SerderKERI.raw of event. - # process event - sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + Steps: + Each pass (walk index table) + For each prefix,sn + For each escrow item dup at prefix,sn: + Get Event + Get and Attach Signatures + Get and Attach Witness Signatures + Process event as if it came in over the wire + If successful then remove from escrow table + """ - # seal source (delegator issuer if any) - delseqner = delsaider = None - couple = self.db.getPde(dgKey(pre, bytes(edig))) - if couple is not None: - delseqner, delsaider = deSourceCouple(couple) + for (epre,), esn, edig in self.db.pdes.getOnItemIter(keys=b''): + try: + #pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(epre, edig) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local source so raise ValidationError which unescrows below + msg = f"PDE Missing escrowed event source at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, - delseqner=delseqner, delsaider=delsaider) + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + msg = f"PDE Missing escrowed event datetime at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - # If process does NOT validate wigs then process will attempt - # to re-escrow and then raise MissingWitnessSignatureError - # (subclass of ValidationError) - # so we can distinquish between ValidationErrors that are - # re-escrow vs non re-escrow. We want process to be idempotent - # with respect to processing events that result in escrow items. - # On re-escrow attempt by process, Pwe escrow is called by - # Kever.self.escrowPWEvent Which calls - # self.db.addPwe(snKey(pre, sn), serder.digb) - # which in turn will NOT enter dig as dup if one already exists. - # So re-escrow attempt will not change the escrowed pwe db. - # Non re-escrow ValidationError means some other issue so unescrow. - # No error at all means processed successfully so also unescrow. - # Assumes that controller signature validation and delegation - # validation will be successful as event would not be in - # partially witnessed escrow unless they had already validated + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPWE): + # escrow stale so raise ValidationError which unescrows below + msg = f"PDE Stale event escrow at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - except MissingWitnessSignatureError as ex: - # still waiting on missing witness sigs - if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery: partial wig escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery: partial wig escrow unescrow failed: %s\n", ex.args[0]) + # get the escrowed event using edig + eraw = self.db.getEvt(dgkey) + if eraw is None: + # no event so so raise ValidationError which unescrows below + msg = f"PDE Missing escrowed evt at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) - except Exception as ex: # log diagnostics errors etc - # error other than waiting on sigs or seal so remove from escrow - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery: partial wig escrow other error on unescrow: %s\n", ex.args[0]) - logger.exception("Kevery: partial wig escrow other error unescrow: %s\n", ex.args[0]) + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event - else: # unescrow succeeded, remove from escrow - # We don't remove all escrows at pre,sn because some might be - # duplicitous so we process remaining escrows in spite of found - # valid event escrow. - self.db.delPwe(snKey(pre, sn), edig) # removes one escrow at key val - logger.trace("Kevery: partial wig escrow unescrow succeeded in valid event: " - "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) - logger.trace("Event Body=\n%s\n", json.dumps(eserder.ked, indent=1)) + # get sigs + sigs = self.db.getSigs(dgkey) # list of sigs + if not sigs: # empty list + # no sigs so raise ValidationError which unescrows below + msg = f"PDE Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.info("Kevery unescrow error: %s", bytes(edig)) + raise ValidationError(msg) + + # get witness signatures (wigs not wits) assumes wont be in this + # escrow if wigs not needed because no wits + wigs = self.db.getWigs(dgkey) # list of wigs if any + # may want to checks wits and wigs here. We are assuming that + # never get to this escrow if wits and not wigs + #if wits and not wigs: # non empty wits but empty wigs + ## wigs maybe empty if not wits or if wits while waiting + ## for first witness signature + ## which may not arrive until some time after event is fully signed + ## so just log for debugging but do not unescrow by raising + ## ValidationError + #logger.info("Kevery unescrow error: Missing event wigs at." + #"dig = %s", edig) + + #raise ValidationError("Missing escrowed evt wigs at " + #"dig = {}.".format(edig)) + + # setup parameters to process event + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # seal source (delegator issuer if any) + # If delegator KEL not available should also cue a trigger to + # get it if still missing when processing escrow. + delseqner = delsaider = None + if (couple := self.db.udes.get(keys=(epre, edig))): + delseqner, delsaider = couple # provided + + #elif eserder.ked["t"] in (Ilks.dip, Ilks.drt,): # walk kel to find + #if eserder.pre in self.kevers: + #delpre = self.kevers[eserder.pre].delpre + #else: + #delpre = eserder.ked["di"] + #seal = dict(i=eserder.ked["i"], s=eserder.snh, d=eserder.said) + #srdr = self.db.findAnchoringSealEvent(pre=delpre, seal=seal) + #if srdr is not None: # found seal in srdr + #delseqner = coring.Seqner(sn=srdr.sn) + #delsaider = coring.Saider(qb64=srdr.said) + #self.db.udes.put(keys=dgkey, val=(delseqner, delsaider)) + + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, + delseqner=delseqner, delsaider=delsaider, + eager=True, local=esr.local) + + # If process does NOT validate delegation then process will attempt + # to re-escrow and then raise MissingDelegationError + # (subclass of ValidationError) + # so we can distinquish between ValidationErrors that are + # re-escrow vs non re-escrow. We want process to be idempotent + # with respect to processing events that result in escrow items. + # On re-escrow attempt by process, Pses escrow is called by + # Kever.self.escrowPDEvent Which calls + # self.db.pdes.add(snKey(pre, sn), serder.digb) + # which in turn will NOT enter dig as dup if one already exists. + # So re-escrow attempt will not change the escrows in db.pdes. + # Non re-escrow ValidationError means some other issue so unescrow. + # No error at all means processed successfully so also unescrow. + # Assumes that controller signature validation and delegation + # validation will be successful as event would not be in + # partially witnessed escrow unless they had already validated + + except MissingDelegationError as ex: + # still waiting on missing delegation source seal + # processEvent idempotently reescrowed + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery PDE unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than waiting on sigs or seal so remove from escrow + # removes one event escrow at key val + self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery PDE unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery PDE unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + # removes one event escrow at key val + self.db.pdes.remOn(keys=epre, on=esn, val=edig) # event idx escrow + self.db.udes.rem(keys=dgkey) # remove source seal escrow if any + logger.info("Kevery partial delegation escrow unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug("Event=\n%s\n", eserder.pretty()) - if ekey == key: # still same so no escrows found on last while iteration - break - key = ekey # setup next while iteration, with key after ekey def processEscrowUnverWitness(self): """ @@ -4982,9 +6100,10 @@ def processEscrowUnverWitness(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, ecouple in self.db.getUweItemsNextIter(key=key): + for ekey, ecouple in self.db.getUweItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow db key + pre, sn = splitSnKey(ekey) # get pre and sn from escrow db key + # get escrowed receipt's rdiger of receipted event and # wiger indexed signature of receipted event rdiger, wiger = deWitnessCouple(ecouple) @@ -4993,8 +6112,8 @@ def processEscrowUnverWitness(self): dtb = self.db.getDts(dgKey(pre, bytes(rdiger.qb64b))) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"Kevery: unver wit escrow unescrow error: Missing event datetime at dig = {rdiger.qb64}" - logger.trace(msg) + msg = f"UWE Missing escrowed event datetime at dig = {rdiger.qb64b}" + logger.trace("Kevery unescrow error: %s", rdiger.qb64b) raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes @@ -5002,8 +6121,8 @@ def processEscrowUnverWitness(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutUWE): # escrow stale so raise ValidationError which unescrows below - msg = f"Kevery: unver wit escrow unescrow error: Stale event escrow at dig = {rdiger.qb64}" - logger.trace(msg) + msg = f"UWE Stale event escrow at dig = {rdiger.qb64b}" + logger.trace("Kevery unescrow error: %s", rdiger.qb64b) raise ValidationError(msg) # lookup database dig of the receipted event in pwes escrow @@ -5015,32 +6134,31 @@ def processEscrowUnverWitness(self): if not found: # no partial witness escrow of event found # so keep in escrow by raising UnverifiedWitnessReceiptError - msg = (f"Kevery: unver wit escrow unescrow error: " - f"Missing witness receipted evt at pre={pre} sn={sn:x}") - logger.trace(msg) + msg = f"UWE Missing witness receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) raise UnverifiedWitnessReceiptError(msg) except UnverifiedWitnessReceiptError as ex: # still waiting on missing prior event to validate # only happens if we process above - if logger.isEnabledFor(logging.TRACE): # adds exception data - logger.trace("Kevery: unver wit escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery: unver wit escrow unescrow failed: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.trace("Kevery: UWE unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: UWE unescrow failed: %s\n", + ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val - if logger.isEnabledFor(logging.TRACE): # adds exception data - logger.trace("Kevery: unver wit escrow other unescrow error on unescrow: %s\n", ex.args[0]) - logger.exception("Kevery: unver wit escrow other unescrow error on unescrow: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.trace("Kevery: UWE other unescrow error: %s\n", ex.args[0]) + logger.exception("Kevery: UWE other unescrow error: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delUwe(snKey(pre, sn), ecouple) # removes one escrow at key val - logger.info("Kevery: unver wit escrow unescrow succeeded for event pre=%s " - "sn=%s", pre, sn) + logger.info("Kevery UWE unescrow succeeded for event pre=%s sn=%s", pre, sn) if ekey == key: # still same so no escrows found on last while iteration break @@ -5096,9 +6214,9 @@ def processEscrowUnverNonTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, etriplet in self.db.getUreItemsNextIter(key=key): + for ekey, etriplet in self.db.getUreItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item rsaider, sprefixer, cigar = deReceiptTriple(etriplet) cigar.verfer = Verfer(qb64b=sprefixer.qb64b) @@ -5106,9 +6224,8 @@ def processEscrowUnverNonTrans(self): dtb = self.db.getDts(dgKey(pre, bytes(rsaider.qb64b))) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = (f"Kevery: unver nontrans escrow unescrow error: " - f"Missing event datetime at dig = {rsaider.qb64}") - logger.trace(msg) + msg = f"URE Missing escrowed event datetime at dig = {rsaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes @@ -5116,9 +6233,8 @@ def processEscrowUnverNonTrans(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutURE): # escrow stale so raise ValidationError which unescrows below - msg = (f"Kevery: unver nontrans escrow unescrow error: " - f"Stale event escrow at dig = {rsaider.qb64}") - logger.trace(msg) + msg = f"URE Stale event escrow at dig = {rsaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # Is receipt for unverified witnessed event in .Pwes escrow @@ -5136,34 +6252,30 @@ def processEscrowUnverNonTrans(self): dig = self.db.getKeLast(snKey(pre, sn)) if dig is None: # no receipted event so keep in escrow - msg = (f"Kevery: unver nontrans escrow unescrow error: " - f"Missing receipted evt at pre={pre} sn={sn:x}") - logger.trace(msg) + msg = f"URE Missing receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) raise UnverifiedReceiptError(msg) # get receipted event using pre and edig raw = self.db.getEvt(dgKey(pre, dig)) if raw is None: # receipted event superseded so remove from escrow - msg = (f"Kevery: unver nontrans escrow unescrow error: " - f"Invalid receipted event reference at pre={pre} sn={sn:x}") - logger.trace(msg) + msg = f"URE Invalid receipted event reference at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) serder = serdering.SerderKERI(raw=bytes(raw)) # receipted event # compare digs if rsaider.qb64b != serder.saidb: - msg = (f"Kevery: unver nontrans escrow unescrow error: Bad receipt dig" - f"pre={pre} sn={sn:x} receiptor={sprefixer.qb64}") - logger.trace(msg) + msg = f"URE Bad escrowed receipt dig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # verify sig verfer key is prefixer from triple if not cigar.verfer.verify(cigar.raw, serder.raw): # no sigs so raise ValidationError which unescrows below - msg = (f"Kevery: unver nontrans escrow unescrow error: " - f"Bad receipt sig at pre={pre} sn={sn:x} receiptor={sprefixer.qb64}") - logger.trace(msg) + msg = f"URE Bad escrowed receipt sig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # get current wits from kever state assuming not stale @@ -5187,28 +6299,138 @@ def processEscrowUnverNonTrans(self): # still waiting on missing prior event to validate # only happens if we process above if logger.isEnabledFor(logging.TRACE): # adds exception data - logger.trace("Kevery: unver nontrans escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery: unver nontrans escrow unescrow failed: %s\n", ex.args[0]) + logger.trace("Kevery: URE escrow other error on unescrow: %s\n", ex.args[0]) + logger.exception("Kevery: URE escrow other error on unescrow: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delUre(snKey(pre, sn), etriplet) # removes one escrow at key val - if logger.isEnabledFor(logging.TRACE): # adds exception data - logger.trace("Kevery: unver nontrans escrow other error on unescrow: %s\n", ex.args[0]) - logger.exception("Kevery: unver nontrans escrow other error on unescrow: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.exception("Kevery URE unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery URE unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delUre(snKey(pre, sn), etriplet) # removes one escrow at key val - logger.info("Kevery: unver nontrans escrow unescrow succeeded for event pre=%s " + logger.info("Kevery UNT unescrow succeeded for event pre=%s " "sn=%s", pre, sn) if ekey == key: # still same so no escrows found on last while iteration break key = ekey # setup next while iteration, with key after ekey + def processEscrowDelegables(self): + """ + Process events escrowed by Kever that require delegation. + + Escrowed items are indexed in database table keyed by prefix and + sn with duplicates given by different dig inserted in insertion order. + This allows FIFO processing of events with same prefix and sn but different + digest. + + Uses .db.delegables.add(key, val) which is IOVal with dups. + + Value is dgkey for event stored in .Evt where .Evt has serder.raw of event. + + Steps: + Each pass (walk index table) + For each prefix,sn + For each escrow item dup at prefix,sn: + Get Event + Get and Attach Signatures + Process event as if it came in over the wire + If successful then remove from escrow table + """ + + for (pre, sn), dig in self.db.delegables.getItemIter(): + try: + edig = dig.encode("utf-8") + dgkey = dgKey(pre.encode("utf-8"), edig) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local source so raise ValidationError which unescrows below + msg = f"DEL Missing escrowed event source at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + + # check date if expired then remove escrow. + dtb = self.db.getDts(dgkey) + if dtb is None: # othewise is a datetime as bytes + # no date time so raise ValidationError which unescrows below + msg = f"DEL Missing escrowed event datetime at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + + # do date math here and discard if stale nowIso8601() bytes + dtnow = helping.nowUTC() + dte = helping.fromIso8601(bytes(dtb)) + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutOOE): + # escrow stale so raise ValidationError which unescrows below + msg = f"DEL Stale event escrow at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + + # get the escrowed event using edig + eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + if eraw is None: + # no event so raise ValidationError which unescrows below + msg = f"DEL Missing escrowed evt at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + + eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event + + # get sigs and attach + sigs = self.db.getSigs(dgKey(pre, bytes(edig))) + if not sigs: # otherwise its a list of sigs + # no sigs so raise ValidationError which unescrows below + msg = f"DEL Missing escrowed evt sigs at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + + sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] + + # get wigs + wigs = self.db.getWigs(dgKey(pre, bytes(edig))) # list of wigs + wigers = [Siger(qb64b=bytes(wig)) for wig in wigs] + + # get delgate seal + couple = self.db.getAes(dgkey) + if couple is not None: # Only try to parse the event if we have the del seal + raw = bytearray(couple) + seqner = coring.Seqner(qb64b=raw, strip=True) + saider = coring.Saider(qb64b=raw) + + # process event + self.processEvent(serder=eserder, sigers=sigers, wigers=wigers, delseqner=seqner, + delsaider=saider, local=esr.local) + else: + raise MissingDelegableApprovalError("No delegation seal found for event.") + + except MissingDelegableApprovalError as ex: + # still waiting on missing delegation approval + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrow failed: %s", ex.args[0]) + + except Exception as ex: # log diagnostics errors etc + # error other than out of order so remove from OO escrow + self.db.delegables.rem(keys=(pre, sn,), val=edig) # removes one escrow at key val + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery unescrowed: %s", ex.args[0]) + else: + logger.error("Kevery unescrowed: %s", ex.args[0]) + + else: # unescrow succeeded, remove from escrow + # We don't remove all escrows at pre,sn because some might be + # duplicitous so we process remaining escrows in spite of found + # valid event escrow. + self.db.delegables.rem(keys=(pre, sn,), val=edig) # removes one escrow at key val + logger.info("Kevery delegables escrow unescrow succeeded in valid event: " + "event=%s", eserder.said) + logger.debug("event=\n%s\n", eserder.pretty()) + def processQueryNotFound(self): """ Process qry events escrowed by Kevery for KELs that have not yet met the criteria of the query. @@ -5220,7 +6442,7 @@ def processQueryNotFound(self): This allows FIFO processing of events with same prefix and sn but different digest. - Uses .db.addQnf(self, key, val) which is IOVal with dups. + Uses .db.qnfs.add(key=(pre, said), val) which is IOVal with dups Value is dgkey for event stored in .Evt where .Evt has serder.raw of event. @@ -5238,15 +6460,15 @@ def processQueryNotFound(self): pre = b'' sn = 0 while True: # break when done - for ekey, edig in self.db.getQnfItemsNextIter(key=key): + for (pre, said), edig in self.db.qnfs.getItemIter(keys=key): try: - pre, _ = splitKey(ekey) # get pre and sn from escrow item # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) + dgkey = dgKey(pre.encode("utf-8"), edig.encode("utf-8")) + dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"Kevery: qnf escrow unescrow error: Missing event datetime at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"QNF Missing escrowed event datetime at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes @@ -5254,34 +6476,40 @@ def processQueryNotFound(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutQNF): # escrow stale so raise ValidationError which unescrows below - msg = f"Kevery: qnf escrow unescrow error: Stale qry event escrow at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"QNF Stale qry event escrow at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # get the escrowed event using edig - eraw = self.db.getEvt(dgKey(pre, bytes(edig))) + eraw = self.db.getEvt(dgkey) if eraw is None: # no event so raise ValidationError which unescrows below - msg = f"Kevery: qnf escrow unescrow error: Missing event at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"QNF Missing escrowed evt at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event # get sigs and attach - sigs = self.db.getSigs(dgKey(pre, bytes(edig))) + sigs = self.db.getSigs(dgkey) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - msg = f"Kevery: qnf escrow unescrow error: Missing event sigs at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"QNF Missing escrowed evt sigs at dig = {bytes(edig)}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # process event sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - # get wigs + # ToDo XXXX get wigs and attach + # getWigs + + # ToDo XXXX get trans endorsements + # getVrcs + + # get nontrans endorsements cigars = [] - cigs = self.db.getRcts(dgKey(pre, bytes(edig))) # list of wigs + cigs = self.db.getRcts(dgkey) # list of wigs for cig in cigs: (_, cigar) = deReceiptCouple(cig) cigars.append(cigar) @@ -5292,23 +6520,23 @@ def processQueryNotFound(self): except QueryNotFoundError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery: qnf escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery: qnf escrow unescrow failed: %s\n", ex.args[0]) + logger.trace("Kevery: QNF escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: QNF escrow unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow - self.db.delQnf(dgKey(pre, edig), edig) # removes one escrow at key val - if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery: qnf other error on unescrow: %s\n", ex.args[0]) - logger.exception("Kevery: qnf other error on unescrow: %s\n", ex.args[0]) + self.db.qnfs.rem(keys=(pre, said), val=edig) # removes one escrow at key val + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Kevery: QNF other unescrow error: %s\n", ex.args[0]) + logger.exception("Kevery: QNF other unescrow error: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. - self.db.delQnf(dgKey(pre, edig), edig) # removes one escrow at key val - logger.info("Kevery: qnf unescrow succeeded in valid event: " - "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) - logger.debug("Event Body=\n%s\n", eserder.pretty()) + self.db.qnfs.rem(keys=(pre, said), val=edig) # removes one escrow at key val + logger.info("Kevery: query not found escrow unescrow succeeded in valid event: " + "key = %s \tdigest = %s", ekey.decode(), edig) + logger.debug("Event=\n%s\n", eserder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break @@ -5399,9 +6627,8 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): elif wiger: # check index and assign verfier to wiger if wiger.index >= len(wits): # bad index # raise ValidationError which removes from escrow by caller - msg = (f"Kevery: find unver escrow unescrow error: Bad escrowed witness receipt" - f" index={wiger.index:i} for pre={pre} sn={sn:x}") - logger.trace(msg) + msg = f"FUV Bad escrowed witness receipt index={wiger.index} at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) wiger.verfer = Verfer(qb64=wits[wiger.index]) @@ -5411,9 +6638,8 @@ def _processEscrowFindUnver(self, pre, sn, rsaider, wiger=None, cigar=None): if found: # verify signature and if verified write to .Wigs if not wiger.verfer.verify(wiger.raw, serder.raw): # not verify # raise ValidationError which unescrows .Uwes or .Ures in caller - msg = (f"Kevery: find unver escrow unescrow error: Bad witness receipt wig." - f" pre={pre} sn={sn:x}") - logger.trace(msg) + msg = f"FUV Bad escrowed witness receipt wig at pre={pre} sn={sn:x}." + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) self.db.addWig(key=dgKey(pre, serder.said), val=wiger.qb64b) # processEscrowPartialWigs removes from this .Pwes escrow @@ -5469,17 +6695,17 @@ def processEscrowUnverTrans(self): ims = bytearray() key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, equinlet in self.db.getVreItemsNextIter(key=key): + for ekey, equinlet in self.db.getVreItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) # check date if expired then remove escrow. dtb = self.db.getDts(dgKey(pre, bytes(esaider.qb64b))) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"Kevery: unver trans escrow unescrow error: Missing event datetime at dig = {esaider.qb64}" - logger.trace(msg) + msg = f"VRE Missing escrowed event datetime at dig = {esaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes @@ -5487,34 +6713,32 @@ def processEscrowUnverTrans(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutVRE): # escrow stale so raise ValidationError which unescrows below - msg = f"Kevery: unver trans escrow unescrow error: Stale event escrow at dig = {esaider.qb64}" - logger.info(msg) + msg = f"VRE Stale event escrow at dig = {esaider.qb64b}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # get dig of the receipted event using pre and sn lastEvt raw = self.db.getKeLast(snKey(pre, sn)) if raw is None: # no event so keep in escrow - msg = f"Kevery: unver trans escrow unescrow error: Missing receipted event at pre={pre} sn={sn:x}" - logger.trace(msg) + msg = f"VRE Missing receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) raise UnverifiedTransferableReceiptError(msg) dig = bytes(raw) # get receipted event using pre and edig raw = self.db.getEvt(dgKey(pre, dig)) if raw is None: # receipted event superseded so remove from escrow - msg = (f"Kevery: unver trans escrow unescrow error: Invalid receipted " - f"event reference at pre={pre} sn={sn:x}") - logger.trace(msg) + msg = f"VRE Invalid receipted evt reference at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) serder = serdering.SerderKERI(raw=bytes(raw)) # receipted event # compare digs if esaider.qb64b != serder.saidb: - msg = (f"Kevery: unver trans escrow unescrow error: Bad escrowed receipt dig at " - f"pre={pre} sn={sn:x} receipter={sprefixer.qb64}") - logger.trace(msg) + msg = f"VRE Bad escrowed receipt dig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # get receipter's last est event @@ -5523,8 +6747,8 @@ def processEscrowUnverTrans(self): sn=sseqner.sn)) if sdig is None: # no event so keep in escrow - msg = f"Kevery: unver trans escrow unescrow error: Missing receipted evt at pre={pre} sn={sn:x}" - logger.trace(msg) + msg = f"VRE Missing receipted evt at pre={pre} sn={sn:x}" + logger.trace("Kevery unescrow error: %s", msg) raise UnverifiedTransferableReceiptError(msg) # retrieve last event itself of receipter @@ -5533,28 +6757,30 @@ def processEscrowUnverTrans(self): sserder = serdering.SerderKERI(raw=bytes(sraw)) if not sserder.compare(said=ssaider.qb64): # seal dig not match event # this unescrows - raise ValidationError("Bad chit seal at sn = {} for rct = {}." - "".format(sseqner.sn, sserder.ked)) + msg = f"VRE Bad chit seal at sn = {sseqner.sn} for rct = {sserder.ked}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # verify sigs and if so write quadruple to database verfers = sserder.verfers if not verfers: - raise ValidationError("Invalid seal est. event dig = {} for " - "receipt from pre ={} no keys." - "".format(ssaider.qb64, sprefixer.qb64)) + msg = (f"VRE Invalid seal est. event dig = {ssaider.qb64} " + f"for receipt from pre = {sprefixer.qb64} no keys") + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) # Set up quadruple sealet = sprefixer.qb64b + sseqner.qb64b + ssaider.qb64b if siger.index >= len(verfers): - raise ValidationError("Index = {} to large for keys." - "".format(siger.index)) + msg = f"VRE Index = {siger.index} too large for keys" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) siger.verfer = verfers[siger.index] # assign verfer if not siger.verfer.verify(siger.raw, serder.raw): # verify sig - msg = (f"Kevery: unver trans escrow unescrow error: bad escrowed trans receipt sig at " - f"pre={pre} sn={sn:x} receipter={sprefixer.qb64}.") - logger.trace(msg) + msg = f"VRE Bad escrowed trans receipt sig at pre={pre} sn={sn:x} receipter={sprefixer.qb64}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # good sig so write receipt quadruple to database @@ -5566,23 +6792,23 @@ def processEscrowUnverTrans(self): # still waiting on missing prior event to validate # only happens if we process above if logger.isEnabledFor(logging.TRACE): # adds exception data - logger.trace("Kevery: unver trans escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery: unver trans escrow unescrow failed: %s\n", ex.args[0]) + logger.trace("Kevery: VRE escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: VRE escrow unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.db.delVre(snKey(pre, sn), equinlet) # removes one escrow at key val - if logger.isEnabledFor(logging.TRACE): # adds exception data - logger.trace("Kevery: unver trans escrow other error on unescrow: %s\n", ex.args[0]) - logger.exception("Kevery: unver trans escrow other error on unescrow: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): # adds exception data + logger.debug("Kevery: VRE escrow other error on unescrow: %s\n", ex.args[0]) + logger.exception("Kevery: VRE escrow other error on unescrow: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delVre(snKey(pre, sn), equinlet) # removes one escrow at key val - logger.info("Kevery: unver trans escrow unescrow succeeded for event = %s", serder.said) - logger.debug("Event Body= \n%s\n", serder.pretty()) + logger.info("Kevery VRE unescrow succeeded for event = %s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break @@ -5626,15 +6852,22 @@ def processEscrowDuplicitous(self): """ key = ekey = b'' # both start same. when not same means escrows found while True: # break when done - for ekey, edig in self.db.getLdeItemsNextIter(key=key): + for ekey, edig in self.db.getLdeItemIter(key=key): try: - pre, sn = splitKeySN(ekey) # get pre and sn from escrow item + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + if not (esr := self.db.esrs.get(keys=dgkey)): # get event source, otherwise error + # no local source so raise ValidationError which unescrows below + msg = f"DUP Missing escrowed event source at dig = {bytes(edig).decode()}" + logger.info("Kevery unescrow error: %s", msg) + raise ValidationError(msg) + # check date if expired then remove escrow. - dtb = self.db.getDts(dgKey(pre, bytes(edig))) + dtb = self.db.getDts(dgkey) if dtb is None: # othewise is a datetime as bytes # no date time so raise ValidationError which unescrows below - msg = f"Kevery: duplicity escrow unescrow error: Missing event datetime at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"DUP Missing escrowed event datetime at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # do date math here and discard if stale nowIso8601() bytes @@ -5642,16 +6875,16 @@ def processEscrowDuplicitous(self): dte = helping.fromIso8601(bytes(dtb)) if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutLDE): # escrow stale so raise ValidationError which unescrows below - msg = f"Kevery: duplicity escrow unescrow error: Stale event escrow at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"DUP Stale event escrow at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) # get the escrowed event using edig eraw = self.db.getEvt(dgKey(pre, bytes(edig))) if eraw is None: # no event so raise ValidationError which unescrows below - msg = f"Kevery: duplicity escrow unescrow error: Missing event at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"DUP Missing escrowed evt at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) eserder = serdering.SerderKERI(raw=bytes(eraw)) # escrowed event @@ -5660,12 +6893,12 @@ def processEscrowDuplicitous(self): sigs = self.db.getSigs(dgKey(pre, bytes(edig))) if not sigs: # otherwise its a list of sigs # no sigs so raise ValidationError which unescrows below - msg = f"Kevery: duplicity escrow unescrow error: Missing event sigs at dig = {bytes(edig)}" - logger.trace(msg) + msg = f"DUP Missing escrowed evt sigs at dig = {bytes(edig).decode()}" + logger.trace("Kevery unescrow error: %s", msg) raise ValidationError(msg) sigers = [Siger(qb64b=bytes(sig)) for sig in sigs] - self.processEvent(serder=eserder, sigers=sigers) + self.processEvent(serder=eserder, sigers=sigers, local=esr.local) # If process does NOT validate event with sigs, becasue it is # still out of order then process will attempt to re-escrow @@ -5684,24 +6917,23 @@ def processEscrowDuplicitous(self): except LikelyDuplicitousError as ex: # still can't determine if duplicitous if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery: duplicity escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Kevery: duplicity escrow unescrow failed: %s\n", ex.args[0]) + logger.trace("Kevery: DUP escrow unescrow failed: %s\n", ex.args[0]) + logger.exception("Kevery: DUP escrow unescrow failed: %s\n",ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than likely duplicitous so remove from escrow self.db.delLde(snKey(pre, sn), edig) # removes one escrow at key val - if logger.isEnabledFor(logging.TRACE): - logger.trace("Kevery: duplicity escrow other error on unescrow: %s\n", ex.args[0]) - logger.exception("Kevery: duplicity escrow other error on unescrow: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.trace("Kevery: DUP escrow other unescrow error: %s\n", ex.args[0]) + logger.exception("Kevery: DUP escrow other unescrow error: %s\n", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.db.delLde(snKey(pre, sn), edig) # removes one escrow at key val - logger.info("Kevery: duplicity escrow unescrow succeeded in valid event: " - "key = %s \tdigest = %s", bytes(ekey).decode(), bytes(edig).decode()) - logger.debug("Event Body=\n%s\n", eserder.pretty()) + logger.info("Kevery duplicitous escrow unescrow succeeded in valid event: event=%s", eserder.said) + logger.debug("event=\n%s\n", eserder.pretty()) if ekey == key: # still same so no escrows found on last while iteration break @@ -5750,7 +6982,7 @@ def loadEvent(db, preb, dig): sigs = db.getSigs(key=dgkey) dsigs = [] for s in sigs: - sig = coring.Siger(qb64b=bytes(s)) + sig = indexing.Siger(qb64b=bytes(s)) dsigs.append(dict(index=sig.index, signature=sig.qb64)) event["signatures"] = dsigs @@ -5762,7 +6994,7 @@ def loadEvent(db, preb, dig): dwigs = [] if wigs := db.getWigs(key=dgkey): for w in wigs: - sig = coring.Siger(qb64b=bytes(w)) + sig = indexing.Siger(qb64b=bytes(w)) dwigs.append(dict(index=sig.index, signature=sig.qb64)) event["witness_signatures"] = dwigs @@ -5784,7 +7016,7 @@ def loadEvent(db, preb, dig): prefix=coring.Prefixer(qb64b=raw, strip=True).qb64, sequence=coring.Seqner(qb64b=raw, strip=True).qb64, said=coring.Saider(qb64b=raw, strip=True).qb64, - signature=coring.Siger(qb64b=raw, strip=True).qb64, + signature=indexing.Siger(qb64b=raw, strip=True).qb64, )) receipts["transferable"] = trans diff --git a/src/keri/core/indexing.py b/src/keri/core/indexing.py new file mode 100644 index 000000000..65c933b5a --- /dev/null +++ b/src/keri/core/indexing.py @@ -0,0 +1,800 @@ +# -*- coding: utf-8 -*- +""" +keri.core.indexing module + +Provides versioning support for Indexer classes and codes +""" +from collections import namedtuple, deque +from dataclasses import dataclass, astuple, asdict +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 + +import pysodium + + +from ..kering import (EmptyMaterialError, RawMaterialError, SoftMaterialError, + InvalidCodeError, InvalidSoftError, + InvalidSizeError, + InvalidCodeSizeError, InvalidVarIndexError, + InvalidVarSizeError, InvalidVarRawSizeError, + ConversionError, InvalidValueError, InvalidTypeError, + ValidationError, VersionError, DerivationError, + EmptyListError, + ShortageError, UnexpectedCodeError, DeserializeError, + UnexpectedCountCodeError, UnexpectedOpCodeError) + +from ..help import helping +from ..help.helping import (sceil, intToB64, b64ToInt, + codeB64ToB2, codeB2ToB64, nabSextets) + + +@dataclass(frozen=True) +class IndexerCodex: + """ IndexerCodex is codex hard (stable) part of all indexer derivation codes. + + Codes indicate which list of keys, current and/or prior next, index is for: + + _Sig: Indices in code may appear in both current signing and + prior next key lists when event has both current and prior + next key lists. Two character code table has only one index + so must be the same for both lists. Other index if for + prior next. + The indices may be different in those code tables which + have two sets of indices. + + _Crt_Sig: Index in code for current signing key list only. + + _Big_: Big index values + + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + + Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. + Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. + ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. + ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. + ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. + Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. + Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. + Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. + Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. + ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. + ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. + Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. + Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. + TBD0: str = '0z' # Test of Var len label L=N*4 <= 4095 char quadlets includes code + TBD1: str = '1z' # Test of index sig lead 1 + TBD4: str = '4z' # Test of index sig lead 1 big + + def __iter__(self): + return iter(astuple(self)) # enables inclusion test with "in" + +IdrDex = IndexerCodex() + + +@dataclass(frozen=True) +class IndexedSigCodex: + """IndexedSigCodex is codex all indexed signature derivation codes. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. + Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. + ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list. + ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. + ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. + Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. + Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. + Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both lists. + Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. + ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. + ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. + Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. + Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. + + def __iter__(self): + return iter(astuple(self)) + +IdxSigDex = IndexedSigCodex() # Make instance + + +@dataclass(frozen=True) +class IndexedCurrentSigCodex: + """IndexedCurrentSigCodex is codex indexed signature codes for current list. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519_Crt_Sig: str = 'B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Crt_Sig: str = 'D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Crt_Sig: str = "F" # ECDSA secp256r1 sig appears in current list. + Ed448_Crt_Sig: str = '0B' # Ed448 signature appears in current list only. + Ed25519_Big_Crt_Sig: str = '2B' # Ed25519 sig appears in current list only. + ECDSA_256k1_Big_Crt_Sig: str = '2D' # ECDSA secp256k1 sig appears in current list only. + ECDSA_256r1_Big_Crt_Sig: str = "2F" # ECDSA secp256r1 sig appears in current list only. + Ed448_Big_Crt_Sig: str = '3B' # Ed448 signature appears in current list only. + + def __iter__(self): + return iter(astuple(self)) + +IdxCrtSigDex = IndexedCurrentSigCodex() # Make instance + + + +@dataclass(frozen=True) +class IndexedBothSigCodex: + """IndexedBothSigCodex is codex indexed signature codes for both lists. + + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Ed25519_Sig: str = 'A' # Ed25519 sig appears same in both lists if any. + ECDSA_256k1_Sig: str = 'C' # ECDSA secp256k1 sig appears same in both lists if any. + ECDSA_256r1_Sig: str = "E" # ECDSA secp256r1 sig appears same in both lists if any. + Ed448_Sig: str = '0A' # Ed448 signature appears in both lists. + Ed25519_Big_Sig: str = '2A' # Ed25519 sig appears in both listsy. + ECDSA_256k1_Big_Sig: str = '2C' # ECDSA secp256k1 sig appears in both lists. + ECDSA_256r1_Big_Sig: str = "2E" # ECDSA secp256r1 sig appears in both lists. + Ed448_Big_Sig: str = '3A' # Ed448 signature appears in both lists. + + def __iter__(self): + return iter(astuple(self)) + +IdxBthSigDex = IndexedBothSigCodex() # Make instance + +# namedtuple for size entries in Incexer derivation code tables +# hs is the hard size int number of chars in hard (stable) part of code +# ss is the soft size int number of chars in soft (unstable) part of code +# os is the other size int number of chars in other index part of soft +# ms = ss - os main index size computed +# fs is the full size int number of chars in code plus appended material if any +# ls is the lead size int number of bytes to pre-pad pre-converted raw binary +Xizage = namedtuple("Xizage", "hs ss os fs ls") + +class Indexer: + """ Indexer is fully qualified cryptographic material primitive base class for + indexed primitives. In special cases some codes in the Index code table + may be of variable length (i.e. not indexed) when the full size table entry + is None. In that case the index is used instread as the length. + + Sub classes are derivation code and key event element context specific. + + Includes the following attributes and properties: + + Attributes: + + Properties: + code is str of stable (hard) part of derivation code + raw (bytes): unqualified crypto material usable for crypto operations + index (int): main index offset into list or length of material + ondex (int | None): other index offset into list or length of material + qb64b (bytes): fully qualified Base64 crypto material + qb64 (str | bytes): fully qualified Base64 crypto material + qb2 (bytes): fully qualified binary crypto material + + Hidden: + ._code (str): value for .code property + ._raw (bytes): value for .raw property + ._index (int): value for .index property + ._ondex (int): value for .ondex property + ._infil is method to compute fully qualified Base64 from .raw and .code + ._binfil is method to compute fully qualified Base2 from .raw and .code + ._exfil is method to extract .code and .raw from fully qualified Base64 + ._bexfil is method to extract .code and .raw from fully qualified Base2 + + """ + # Hards table maps from bytes Base64 first code char to int of hard size, hs, + # (stable) of code. The soft size, ss, (unstable) is always > 0 for Indexer. + Hards = ({chr(c): 1 for c in range(65, 65 + 26)}) + Hards.update({chr(c): 1 for c in range(97, 97 + 26)}) + Hards.update([('0', 2), ('1', 2), ('2', 2), ('3', 2), ('4', 2)]) + # Sizes table maps hs chars of code to Xizage namedtuple of (hs, ss, os, fs, ls) + # where hs is hard size, ss is soft size, os is other index size, + # and fs is full size, ls is lead size. + # where ss includes os, so main index size ms = ss - os + # soft size, ss, should always be > 0 for Indexer + Sizes = { + 'A': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'B': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'C': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'D': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'E': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'F': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + '0A': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), + '0B': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), + '2A': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2B': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2C': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2D': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2E': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2F': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '3A': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), + '3B': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), + '0z': Xizage(hs=2, ss=2, os=0, fs=None, ls=0), + '1z': Xizage(hs=2, ss=2, os=1, fs=76, ls=1), + '4z': Xizage(hs=2, ss=6, os=3, fs=80, ls=1), + } + # Bards table maps to hard size, hs, of code from bytes holding sextets + # converted from first code char. Used for ._bexfil. + Bards = ({codeB64ToB2(c): hs for c, hs in Hards.items()}) + + Codes = asdict(IdrDex) # map code name to code + Names = {val : key for key, val in Codes.items()} # invert map code to code name + + + + def __init__(self, raw=None, code=IdrDex.Ed25519_Sig, index=0, ondex=None, + qb64b=None, qb64=None, qb2=None, strip=False, **kwa): + """ + Validate as fully qualified + Parameters: + raw (bytes): unqualified crypto material usable for crypto operations + code is str of stable (hard) part of derivation code + index (int): main index offset into list or length of material + ondex (int | None): other index offset into list or length of material + qb64b (bytes): fully qualified Base64 crypto material + qb64 (str | bytes): fully qualified Base64 crypto material + qb2 (bytes): fully qualified binary crypto material + strip (bool): True means strip counter contents from input stream + bytearray after parsing qb64b or qb2. False means do not strip + + Needs either (raw and code and index) or qb64b or qb64 or qb2 + Otherwise raises EmptyMaterialError + When raw and code provided then validate that code is correct + for length of raw and assign .raw + Else when qb64b or qb64 or qb2 provided extract and assign + .raw, .code, .index, .ondex. + + """ + if raw is not None: # raw provided + if not code: + raise EmptyMaterialError("Improper initialization need either " + "(raw and code) or qb64b or qb64 or qb2.") + if not isinstance(raw, (bytes, bytearray)): + raise TypeError(f"Not a bytes or bytearray, raw={raw}.") + + if code not in self.Sizes: + raise UnexpectedCodeError(f"Unsupported code={code}.") + + hs, ss, os, fs, ls = self.Sizes[code] # get sizes for code + cs = hs + ss # both hard + soft code size + ms = ss - os + + if not isinstance(index, int) or index < 0 or index > (64 ** ms - 1): + raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") + + if isinstance(ondex, int) and os and not (ondex >= 0 and ondex <= (64 ** os - 1)): + raise InvalidVarIndexError(f"Invalid ondex={ondex} for code={code}.") + + if code in IdxCrtSigDex and ondex is not None: + raise InvalidVarIndexError(f"Non None ondex={ondex} for code={code}.") + + if code in IdxBthSigDex: + if ondex is None: # set default + ondex = index # when not provided make ondex match index + else: + if ondex != index and os == 0: # must match if os == 0 + raise InvalidVarIndexError(f"Non matching ondex={ondex}" + f" and index={index} for " + f"code={code}.") + + + if not fs: # compute fs from index + if cs % 4: + raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise InvalidCodeSizeError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + rawsize = (fs - cs) * 3 // 4 + + raw = raw[:rawsize] # copy rawsize from stream, may be less + if len(raw) != rawsize: # forbids shorter + raise RawMaterialError(f"Not enougth raw bytes for code={code}" + f"and index={index} ,expected {rawsize} " + f"got {len(raw)}.") + + self._code = code + self._index = index + self._ondex = ondex + self._raw = bytes(raw) # crypto ops require bytes not bytearray + + elif qb64b is not None: + self._exfil(qb64b) + if strip: # assumes bytearray + del qb64b[:len(self.qb64b)] # may be variable length fs + + elif qb64 is not None: + self._exfil(qb64) + + elif qb2 is not None: + self._bexfil(qb2) + if strip: # assumes bytearray + del qb2[:len(self.qb2)] # may be variable length fs + + else: + raise EmptyMaterialError("Improper initialization need either " + "(raw and code and index) or qb64b or " + "qb64 or qb2.") + + @classmethod + def _rawSize(cls, code): + """ + Returns expected raw size in bytes for a given code. Not applicable to + codes with fs = None + """ + hs, ss, os, fs, ls = cls.Sizes[code] # get sizes + return ((fs - (hs + ss)) * 3 // 4) + + @property + def code(self): + """ + Returns ._code + Makes .code read only + """ + return self._code + + + @property + def name(self): + """ + Returns: + name (str): code name for self.code. Used for annotation for + primitives like Matter + + """ + return self.Names[self.code] + + @property + def raw(self): + """ + Returns ._raw + Makes .raw read only + """ + return self._raw + + @property + def index(self): + """ + Returns ._index + Makes .index read only + """ + return self._index + + @property + def ondex(self): + """ + Returns ._ondex + Makes .ondex read only + """ + return self._ondex + + @property + def qb64b(self): + """ + Property qb64b: + Returns Fully Qualified Base64 Version encoded as bytes + Assumes self.raw and self.code are correctly populated + """ + return self._infil() + + @property + def qb64(self): + """ + Property qb64: + Returns Fully Qualified Base64 Version + Assumes self.raw and self.code are correctly populated + """ + return self.qb64b.decode("utf-8") + + @property + def qb2(self): + """ + Property qb2: + Returns Fully Qualified Binary Version Bytes + """ + return self._binfil() + + def _infil(self): + """ + Returns fully qualified attached sig base64 bytes computed from + self.raw, self.code and self.index. + + cs = hs + ss + os = ss - ms (main index size) + when fs None then size computed & fs = size * 4 + cs + + """ + code = self.code # codex value chars hard code + index = self.index # main index value + ondex = self.ondex # other index value + raw = self.raw # bytes or bytearray + + ps = (3 - (len(raw) % 3)) % 3 # if lead then same pad size chars & lead size bytes + hs, ss, os, fs, ls = self.Sizes[code] + cs = hs + ss + ms = ss - os + + if not fs: # compute fs from index + if cs % 4: + raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise InvalidCodeSizeError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + if index < 0 or index > (64 ** ms - 1): + raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") + + if (isinstance(ondex, int) and os and + not (ondex >= 0 and ondex <= (64 ** os - 1))): + raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " + f"code={code}.") + + # both is hard code + converted index + converted ondex + both = (f"{code}{intToB64(index, l=ms)}" + f"{intToB64(ondex if ondex is not None else 0, l=os)}") + + # check valid pad size for whole code size, assumes ls is zero + if len(both) != cs: + raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." + .format(cs, len(both))) + + if (cs % 4) != ps - ls: # adjusted pad given lead bytes + raise InvalidCodeSizeError(f"Invalid code={both} for converted" + f" raw pad size={ps}.") + + # prepend pad bytes, convert, then replace pad chars with full derivation + # code including index, + full = both.encode("utf-8") + encodeB64(bytes([0] * ps) + raw)[ps - ls:] + + if len(full) != fs: # invalid size + raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") + + return full + + + def _binfil(self): + """ + Returns bytes of fully qualified base2 bytes, that is .qb2 + self.code and self.index converted to Base2 + self.raw left shifted + with pad bits equivalent of Base64 decode of .qb64 into .qb2 + """ + code = self.code # codex chars hard code + index = self.index # main index value + ondex = self.ondex # other index value + raw = self.raw # bytes or bytearray + + ps = (3 - (len(raw) % 3)) % 3 # same pad size chars & lead size bytes + hs, ss, os, fs, ls = self.Sizes[code] + cs = hs + ss + ms = ss - os + + if index < 0 or index > (64 ** ss - 1): + raise InvalidVarIndexError(f"Invalid index={index} for code={code}.") + + if (isinstance(ondex, int) and os and + not (ondex >= 0 and ondex <= (64 ** os - 1))): + raise InvalidVarIndexError(f"Invalid ondex={ondex} for os={os} and " + f"code={code}.") + + if not fs: # compute fs from index + if cs % 4: + raise InvalidCodeSizeError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise InvalidCodeSizeError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + # both is hard code + converted index + both = (f"{code}{intToB64(index, l=ms)}" + f"{intToB64(ondex if ondex is not None else 0, l=os)}") + + if len(both) != cs: + raise InvalidCodeSizeError("Mismatch code size = {} with table = {}." + .format(cs, len(both))) + + if (cs % 4) != ps - ls: # adjusted pad given lead bytes + raise InvalidCodeSizeError(f"Invalid code={both} for converted" + f" raw pad size={ps}.") + + n = sceil(cs * 3 / 4) # number of b2 bytes to hold b64 code + index + # convert code both to right align b2 int then left shift in pad bits + # then convert to bytes + bcode = (b64ToInt(both) << (2 * (ps - ls))).to_bytes(n, 'big') + full = bcode + bytes([0] * ls) + raw + + bfs = len(full) # binary full size + if bfs % 3 or (bfs * 4 // 3) != fs: # invalid size + raise InvalidCodeSizeError(f"Invalid code={both} for raw size={len(raw)}.") + + return full + + + def _exfil(self, qb64b): + """ + Extracts self.code, self.index, and self.raw from qualified base64 bytes qb64b + + cs = hs + ss + ms = ss - os (main index size) + when fs None then size computed & fs = size * 4 + cs + """ + if not qb64b: # empty need more bytes + raise ShortageError("Empty material.") + + first = qb64b[:1] # extract first char code selector + if hasattr(first, "decode"): + first = first.decode("utf-8") + if first not in self.Hards: + if first[0] == '-': + raise UnexpectedCountCodeError("Unexpected count code start" + "while extracing Indexer.") + elif first[0] == '_': + raise UnexpectedOpCodeError("Unexpected op code start" + "while extracing Indexer.") + else: + raise UnexpectedCodeError(f"Unsupported code start char={first}.") + + hs = self.Hards[first] # get hard code size + if len(qb64b) < hs: # need more bytes + raise ShortageError(f"Need {hs - len(qb64b)} more characters.") + + hard = qb64b[:hs] # get hard code + if hasattr(hard, "decode"): + hard = hard.decode("utf-8") + if hard not in self.Sizes: + raise UnexpectedCodeError(f"Unsupported code ={hard}.") + + hs, ss, os, fs, ls = self.Sizes[hard] # assumes hs in both tables consistent + cs = hs + ss # both hard + soft code size + ms = ss - os + # assumes that unit tests on Indexer and IndexerCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) + # assumes no variable length indexed codes so fs is not None + + if len(qb64b) < cs: # need more bytes + raise ShortageError(f"Need {cs - len(qb64b)} more characters.") + + index = qb64b[hs:hs+ms] # extract index/size chars + if hasattr(index, "decode"): + index = index.decode("utf-8") + index = b64ToInt(index) # compute int index + + ondex = qb64b[hs+ms:hs+ms+os] # extract ondex chars + if hasattr(ondex, "decode"): + ondex = ondex.decode("utf-8") + + if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 + ondex = b64ToInt(ondex) if os else None # compute ondex from code + if ondex: # not zero or None so error + raise ValueError(f"Invalid ondex={ondex} for code={hard}.") + else: + ondex = None # zero so set to None when current only + else: + ondex = b64ToInt(ondex) if os else index + + # index is index for some codes and variable length for others + if not fs: # compute fs from index which means variable length + if cs % 4: + raise ValidationError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise ValidationError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + if len(qb64b) < fs: # need more bytes + raise ShortageError(f"Need {fs - len(qb64b)} more chars.") + + qb64b = qb64b[:fs] # fully qualified primitive code plus material + if hasattr(qb64b, "encode"): # only convert extracted chars from stream + qb64b = qb64b.encode("utf-8") + + # strip off prepended code and append pad characters + #ps = cs % 4 # pad size ps = cs mod 4, same pad chars and lead bytes + #base = ps * b'A' + qb64b[cs:] # replace prepend code with prepad zeros + #raw = decodeB64(base)[ps+ls:] # decode and strip off ps+ls prepad bytes + + # check for non-zeroed pad bits or lead bytes + ps = cs % 4 # code pad size ps = cs mod 4 + pbs = 2 * (ps if ps else ls) # pad bit size in bits + if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls + base = ps * b'A' + qb64b[cs:] # replace pre code with prepad chars of zero + paw = decodeB64(base) # decode base to leave prepadded raw + pi = (int.from_bytes(paw[:ps], "big")) # prepad as int + if pi & (2 ** pbs - 1 ): # masked pad bits non-zero + raise ValueError(f"Non zeroed prepad bits = " + f"{pi & (2 ** pbs - 1 ):<06b} in {qb64b[cs:cs+1]}.") + raw = paw[ps:] # strip off ps prepad paw bytes + else: # not ps. IF not ps THEN may or may not be ls (lead) + base = qb64b[cs:] # strip off code leaving lead chars if any and value + # decode lead chars + val leaving lead bytes + raw bytes + # then strip off ls lead bytes leaving raw + paw = decodeB64(base) # decode base to leave prepadded paw bytes + li = int.from_bytes(paw[:ls], "big") # lead as int + if li: # pre pad lead bytes must be zero + if ls == 1: + raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") + else: + raise ValueError(f"Non zeroed lead bytes = 0x{li:04x}.") + + raw = paw[ls:] + + if len(raw) != (len(qb64b) - cs) * 3 // 4: # exact lengths + raise ConversionError(f"Improperly qualified material = {qb64b}") + + self._code = hard + self._index = index + self._ondex = ondex + self._raw = raw # must be bytes for crpto opts and immutable not bytearray + + + + def _bexfil(self, qb2): + """ + Extracts self.code, self.index, and self.raw from qualified base2 bytes qb2 + + cs = hs + ss + ms = ss - os (main index size) + when fs None then size computed & fs = size * 4 + cs + """ + if not qb2: # empty need more bytes + raise ShortageError("Empty material, Need more bytes.") + + first = nabSextets(qb2, 1) # extract first sextet as code selector + if first not in self.Bards: + if first[0] == b'\xf8': # b64ToB2('-') + raise UnexpectedCountCodeError("Unexpected count code start" + "while extracing Matter.") + elif first[0] == b'\xfc': # b64ToB2('_') + raise UnexpectedOpCodeError("Unexpected op code start" + "while extracing Matter.") + else: + raise UnexpectedCodeError(f"Unsupported code start sextet={first}.") + + hs = self.Bards[first] # get code hard size equvalent sextets + bhs = sceil(hs * 3 / 4) # bhs is min bytes to hold hs sextets + if len(qb2) < bhs: # need more bytes + raise ShortageError(f"Need {bhs - len(qb2)} more bytes.") + + hard = codeB2ToB64(qb2, hs) # extract and convert hard part of code + if hard not in self.Sizes: + raise UnexpectedCodeError(f"Unsupported code ={hard}.") + + hs, ss, os, fs, ls = self.Sizes[hard] + cs = hs + ss # both hs and ss + ms = ss - os + # assumes that unit tests on Indexer and IndexerCodex ensure that + # .Codes and .Sizes are well formed. + # hs consistent and hs > 0 and ss > 0 and (fs >= hs + ss if fs is not None else True) + + bcs = sceil(cs * 3 / 4) # bcs is min bytes to hold cs sextets + if len(qb2) < bcs: # need more bytes + raise ShortageError("Need {} more bytes.".format(bcs - len(qb2))) + + both = codeB2ToB64(qb2, cs) # extract and convert both hard and soft part of code + index = b64ToInt(both[hs:hs+ms]) # compute index + + if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 + ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else None # compute ondex from code + if ondex: # not zero or None so error + raise ValueError(f"Invalid ondex={ondex} for code={hard}.") + else: + ondex = None # zero so set to None when current only + else: + ondex = b64ToInt(both[hs+ms:hs+ms+os]) if os else index + + if hard in IdxCrtSigDex: # if current sig then ondex from code must be 0 + if ondex: # not zero so error + raise ValueError(f"Invalid ondex={ondex} for code={hard}.") + else: # zero so set to None + ondex = None + + if not fs: # compute fs from size chars in ss part of code + if cs % 4: + raise ValidationError(f"Whole code size not multiple of 4 for " + f"variable length material. cs={cs}.") + if os != 0: + raise ValidationError(f"Non-zero other index size for " + f"variable length material. os={os}.") + fs = (index * 4) + cs + + bfs = sceil(fs * 3 / 4) # bfs is min bytes to hold fs sextets + if len(qb2) < bfs: # need more bytes + raise ShortageError("Need {} more bytes.".format(bfs - len(qb2))) + + qb2 = qb2[:bfs] # extract qb2 fully qualified primitive code plus material + + # check for non-zeroed prepad bits or lead bytes + ps = cs % 4 # code pad size ps = cs mod 4 + pbs = 2 * (ps if ps else ls) # pad bit size in bits + if ps: # ps. IF ps THEN not ls (lead) and vice versa OR not ps and not ls + # convert last byte of code bytes in which are pad bits to int + pi = (int.from_bytes(qb2[bcs-1:bcs], "big")) + if pi & (2 ** pbs - 1 ): # masked pad bits non-zero + raise ValueError(f"Non zeroed pad bits = " + f"{pi & (2 ** pbs - 1 ):>08b} in 0x{pi:02x}.") + else: # not ps. IF not ps THEN may or may not be ls (lead) + li = int.from_bytes(qb2[bcs:bcs+ls], "big") # lead as int + if li: # pre pad lead bytes must be zero + if ls == 1: + raise ValueError(f"Non zeroed lead byte = 0x{li:02x}.") + else: + raise ValueError(f"Non zeroed lead bytes = 0x{li:02x}.") + + + raw = qb2[(bcs + ls):] # strip code and leader bytes from qb2 to get raw + + if len(raw) != (len(qb2) - bcs - ls): # exact lengths + raise ConversionError(f"Improperly qualified material = {qb2}") + + self._code = hard + self._index = index + self._ondex = ondex + self._raw = bytes(raw) # must be bytes for crypto ops and not bytearray mutable + + +class Siger(Indexer): + """ + Siger is subclass of Indexer, indexed signature material, + + Adds .verfer property which is instance of Verfer that provides + associated signature verifier. + + See Indexer for inherited attributes and properties: + + Attributes: + + Properties: + verfer (Verfer): instance if any provides public verification key + + Methods: + + Hidden: + _verfer (Verfer): value for .verfer property + + + """ + + def __init__(self, verfer=None, **kwa): + """Initialze instance + + Parameters: See Matter for inherted parameters + verfer (Verfer): instance if any provides public verification key + + """ + super(Siger, self).__init__(**kwa) + if self.code not in IdxSigDex: + raise ValidationError("Invalid code = {} for Siger." + "".format(self.code)) + self.verfer = verfer + + @property + def verfer(self): + """ + Property verfer: + Returns Verfer instance + Assumes ._verfer is correctly assigned + """ + return self._verfer + + @verfer.setter + def verfer(self, verfer): + """ verfer property setter """ + self._verfer = verfer + + diff --git a/src/keri/core/parsing.py b/src/keri/core/parsing.py index 145bbec46..98fe18e4b 100644 --- a/src/keri/core/parsing.py +++ b/src/keri/core/parsing.py @@ -6,66 +6,20 @@ """ import logging -from collections import namedtuple -from dataclasses import dataclass, astuple -from .coring import (Ilks, CtrDex, Counter, Seqner, Siger, Cigar, - Dater, Verfer, Prefixer, Saider, Pather, Protos ) +from ..kering import Vrsn_1_0, Vrsn_2_0 +from .coring import (Ilks, Seqner, Cigar, + Dater, Verfer, Prefixer, Saider, Pather, Matter) +from .counting import Counter, Codens, CtrDex_1_0 +from .indexing import (Siger, ) from . import serdering from .. import help from .. import kering +from ..kering import Colds, sniff, Vrsn_1_0, Vrsn_2_0 logger = help.ogler.getLogger() -@dataclass(frozen=True) -class ColdCodex: - """ - ColdCodex is codex of cold stream start tritets of first byte - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - - First three bits: - 0o0 = 000 free - 0o1 = 001 cntcode B64 - 0o2 = 010 opcode B64 - 0o3 = 011 json - 0o4 = 100 mgpk - 0o5 = 101 cbor - 0o6 = 110 mgpk - 007 = 111 cntcode or opcode B2 - - status is one of ('evt', 'txt', 'bny' ) - 'evt' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) - 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) - 'bny' if tritet in (ColdDex.CtOpB2,) - - otherwise raise ColdStartError - - x = bytearray([0x2d, 0x5f]) - x == bytearray(b'-_') - x[0] >> 5 == 0o1 - True - """ - Free: int = 0o0 # not taken - CtB64: int = 0o1 # CountCode Base64 - OpB64: int = 0o2 # OpCode Base64 - JSON: int = 0o3 # JSON Map Event Start - MGPK1: int = 0o4 # MGPK Fixed Map Event Start - CBOR: int = 0o5 # CBOR Map Event Start - MGPK2: int = 0o6 # MGPK Big 16 or 32 Map Event Start - CtOpB2: int = 0o7 # CountCode or OpCode Base2 - - def __iter__(self): - return iter(astuple(self)) - - -ColdDex = ColdCodex() # Make instance - -Coldage = namedtuple("Coldage", 'msg txt bny') # stream cold start status -Colds = Coldage(msg='msg', txt='txt', bny='bny') - - class Parser: """ Parser is stream parser that processes an incoming message stream. @@ -88,10 +42,16 @@ class Parser: whenever stream includes pipelined count codes. kvy (Kevery): route KEL message types to this instance tvy (Tevery): route TEL message types to this instance + exc (Exchanger): route EXN message types to this instance + rvy (Revery): reply (RPY) message handler + vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation """ - def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, + tvy=None, exc=None, rvy=None, vry=None, local=False): """ Initialize instance: @@ -106,6 +66,8 @@ def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex exc (Exchanger): route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation """ self.ims = ims if ims is not None else bytearray() self.framed = True if framed else False # extract until end-of-stream @@ -115,46 +77,8 @@ def __init__(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex self.exc = exc self.rvy = rvy self.vry = vry + self.local = True if local else False - @staticmethod - def sniff(ims): - """ - Returns status string of cold start of stream ims bytearray by looking - at first triplet of first byte to determin if message or counter code - and if counter code whether Base64 or Base2 representation - - First three bits: - 0o0 = 000 free - 0o1 = 001 cntcode B64 - 0o2 = 010 opcode B64 - 0o3 = 011 json - 0o4 = 100 mgpk - 0o5 = 101 cbor - 0o6 = 110 mgpk - 007 = 111 cntcode or opcode B2 - - counter B64 in (0o1, 0o2) return 'txt' - counter B2 in (0o7) return 'bny' - event in (0o3, 0o4, 0o5, 0o6) return 'evt' - unexpected in (0o0) raise ColdStartError - Colds = Coldage(msg='msg', txt='txt', bny='bny') - - 'msg' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) - 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) - 'bny' if tritet in (ColdDex.CtOpB2,) - """ - if not ims: - raise kering.ShortageError("Need more bytes.") - - tritet = ims[0] >> 5 - if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2): - return Colds.msg - if tritet in (ColdDex.CtB64, ColdDex.OpB64): - return Colds.txt - if tritet in (ColdDex.CtOpB2,): - return Colds.bny - - raise kering.ColdStartError("Unexpected tritet={} at stream start.".format(tritet)) @staticmethod def extract(ims, klas, cold=Colds.txt): @@ -170,8 +94,9 @@ def extract(ims, klas, cold=Colds.txt): else: raise kering.ColdStartError("Invalid stream state cold={}.".format(cold)) + @staticmethod - def _extractor(ims, klas, cold=Colds.txt, abort=False): + def _extractor(ims, klas, cold=Colds.txt, abort=False, gvrsn=Vrsn_1_0): """ Returns generator to extract and return instance of klas from input message stream, ims, given stream state, cold, is txt or bny. @@ -180,6 +105,14 @@ def _extractor(ims, klas, cold=Colds.txt, abort=False): Inits klas from ims using qb64b or qb2 parameter based on cold. Yields if not enough bytes in ims to fill out klas instance. + Parameters: + ims (bytearray): input message stream (must be strippable) + klas (Serder | Counter | Matter | Indexer): subclass that is parsable + cold (Coldage): instance str value + abort (bool): True means abort if bad pipelined frame Shortage + False means do not abort if Shortage just wait for more + gvrsn (Versionage): instance of genera version of CESR code tables + Usage: instance = self._extractor @@ -187,9 +120,9 @@ def _extractor(ims, klas, cold=Colds.txt, abort=False): while True: try: if cold == Colds.txt: - return klas(qb64b=ims, strip=True) + return klas(qb64b=ims, strip=True, gvrsn=gvrsn) elif cold == Colds.bny: - return klas(qb2=ims, strip=True) + return klas(qb2=ims, strip=True, gvrsn=gvrsn) else: raise kering.ColdStartError("Invalid stream state cold={}.".format(cold)) except kering.ShortageError as ex: @@ -197,6 +130,7 @@ def _extractor(ims, klas, cold=Colds.txt, abort=False): raise # bad pipelined frame so abort by raising error yield + def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False): """ @@ -213,10 +147,10 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) Returns: """ - if ctr.code != CtrDex.SadPathSig: + if ctr.code != CtrDex_1_0.SadPathSigGroups: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ctr.code, CtrDex.ControllerIdxSigs)) + "".format(ctr.code, CtrDex_1_0.ControllerIdxSigs)) subpath = yield from self._extractor(ims, klas=Pather, @@ -229,10 +163,10 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) klas=Counter, cold=cold, abort=pipelined) - if sctr.code == CtrDex.TransIdxSigGroups: + if sctr.code == CtrDex_1_0.TransIdxSigGroups: for prefixer, seqner, saider, isigers in self._transIdxSigGroups(sctr, ims, cold=cold, pipelined=pipelined): yield sctr.code, (subpath, prefixer, seqner, saider, isigers) - elif sctr.code == CtrDex.ControllerIdxSigs: + elif sctr.code == CtrDex_1_0.ControllerIdxSigs: isigers = [] for i in range(sctr.count): # extract each attached signature isiger = yield from self._extractor(ims=ims, @@ -241,13 +175,14 @@ def _sadPathSigGroup(self, ctr, ims, root=None, cold=Colds.txt, pipelined=False) abort=pipelined) isigers.append(isiger) yield sctr.code, (subpath, isigers) - elif sctr.code == CtrDex.NonTransReceiptCouples: + elif sctr.code == CtrDex_1_0.NonTransReceiptCouples: for cigar in self._nonTransReceiptCouples(ctr=sctr, ims=ims, cold=cold, pipelined=pipelined): yield sctr.code, (subpath, cigar) else: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ctr.code, CtrDex.ControllerIdxSigs)) + "".format(ctr.code, CtrDex_1_0.ControllerIdxSigs)) + def _transIdxSigGroups(self, ctr, ims, cold=Colds.txt, pipelined=False): """ @@ -289,10 +224,10 @@ def _transIdxSigGroups(self, ctr, ims, cold=Colds.txt, pipelined=False): klas=Counter, cold=cold, abort=pipelined) - if ictr.code != CtrDex.ControllerIdxSigs: + if ictr.code != CtrDex_1_0.ControllerIdxSigs: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ictr.code, CtrDex.ControllerIdxSigs)) + "".format(ictr.code, CtrDex_1_0.ControllerIdxSigs)) isigers = [] for i in range(ictr.count): # extract each attached signature isiger = yield from self._extractor(ims=ims, @@ -303,6 +238,7 @@ def _transIdxSigGroups(self, ctr, ims, cold=Colds.txt, pipelined=False): yield prefixer, seqner, saider, isigers + def _nonTransReceiptCouples(self, ctr, ims, cold=Colds.txt, pipelined=False): """ Extract attached rct couplets into list of sigvers @@ -335,7 +271,9 @@ def _nonTransReceiptCouples(self, ctr, ims, cold=Colds.txt, pipelined=False): yield cigar - def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, + exc=None, rvy=None, vry=None, local=None, gvrsn=Vrsn_1_0): """ Processes all messages from incoming message stream, ims, when provided. Otherwise process messages from .ims @@ -358,11 +296,19 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=No exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local + gvrsn (Versionage): instance of genera version of CESR code tables New Logic: Attachments must all have counters so know if txt or bny format for attachments. So even when framed==True must still have counters. """ + local = local if local is not None else self.local + local = True if local else False + + parsator = self.allParsator(ims=ims, framed=framed, pipeline=pipeline, @@ -370,7 +316,9 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=No tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local, + gvrsn=gvrsn) while True: try: @@ -378,7 +326,9 @@ def parse(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=No except StopIteration: break - def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, + exc=None, rvy=None, vry=None, local=None): """ Processes one messages from incoming message stream, ims, when provided. Otherwise process message from .ims @@ -400,11 +350,17 @@ def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex tvy (Tevery): route TEL message types to this instance exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local New Logic: Attachments must all have counters so know if txt or bny format for attachments. So even when framed==True must still have counters. """ + local = local if local is not None else self.local + local = True if local else False + parsator = self.onceParsator(ims=ims, framed=framed, pipeline=pipeline, @@ -412,14 +368,18 @@ def parseOne(self, ims=None, framed=True, pipeline=False, kvy=None, tvy=None, ex tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) while True: try: next(parsator) except StopIteration: break - def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, + tvy=None, exc=None, rvy=None, vry=None, local=None, + gvrsn=Vrsn_1_0): """ Returns generator to parse all messages from incoming message stream, ims until ims is exhausted (empty) then returns. @@ -442,6 +402,10 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local + gvrsn (Versionage): instance of genera version of CESR code tables New Logic: Attachments must all have counters so know if txt or bny format for @@ -460,6 +424,8 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc = exc if exc is not None else self.exc rvy = rvy if rvy is not None else self.rvy vry = vry if vry is not None else self.vry + local = local if local is not None else self.local + local = True if local else False while ims: # only process until ims empty try: @@ -470,34 +436,38 @@ def allParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local, + gvrsn=gvrsn) except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser sized group error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser sized group error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg extraction error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser msg extraction error: %s", ex.args[0]) del ims[:] # delete rest of stream to force cold restart except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg non-extraction error: %s\n", ex) - else: - logger.error("Parser msg non-extraction error: %s\n", ex) + if logger.isEnabledFor(logging.TRACE): + logger.exception("Parser msg validation or non-extraction error: %s", ex) + if logger.isEnabledFor(logging.DEBUG): + logger.error("Parser msg validation or non-extraction error: %s", ex) yield return True - def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, + tvy=None, exc=None, rvy=None, vry=None, local=None): """ Returns generator to parse one message from incoming message stream, ims. If ims not provided parse messages from .ims @@ -518,6 +488,9 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verfifier): credential verifier with wallet storage + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local New Logic: Attachments must all have counters so know if txt or bny format for @@ -536,6 +509,8 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc = exc if exc is not None else self.exc rvy = rvy if rvy is not None else self.rvy vry = vry if vry is not None else self.vry + local = local if local is not None else self.local + local = True if local else False done = False while not done: @@ -547,41 +522,44 @@ def onceParsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream - if logger.isEnabledFor(logging.ERROR): - logger.exception("Kevery msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery msg extraction error: %s", ex.args[0]) else: - logger.error("Kevery msg extraction error: %s\n", ex.args[0]) + logger.error("Kevery msg extraction error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error - if logger.isEnabledFor(logging.ERROR): - logger.exception("Kevery msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery msg extraction error: %s", ex.args[0]) else: - logger.error("Kevery msg extraction error: %s\n", ex.args[0]) + logger.error("Kevery msg extraction error: %s", ex.args[0]) del ims[:] # delete rest of stream to force cold restart except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume - if logger.isEnabledFor(logging.ERROR): - logger.exception("Kevery msg non-extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Kevery msg non-extraction error: %s", ex) else: - logger.error("Kevery msg non-extraction error: %s\n", ex.args[0]) + logger.error("Kevery msg non-extraction error: %s", ex) finally: done = True return done - def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc=None, rvy=None, vry=None): + + def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, + exc=None, rvy=None, vry=None, local=None): """ Returns generator to continually parse messages from incoming message - stream, ims. Empty yields when ims is emply. + stream, ims. Empty yields when ims is emply. Does not return. Useful for always running servers. One yield from per each message if any. - Continually yields while ims is empty. + Continually yields while ims is empty, i.e. does not return. If ims not provided then parse messages from .ims Parameters: @@ -600,6 +578,10 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verifier): credential processor + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local + New Logic: Attachments must all have counters so know if txt or bny format for @@ -618,6 +600,8 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc exc = exc if exc is not None else self.exc rvy = rvy if rvy is not None else self.rvy vry = vry if vry is not None else self.vry + local = local if local is not None else self.local + local = True if local else False while True: # continuous stream processing never stop try: @@ -628,36 +612,38 @@ def parsator(self, ims=None, framed=None, pipeline=None, kvy=None, tvy=None, exc tvy=tvy, exc=exc, rvy=rvy, - vry=vry) + vry=vry, + local=local) except kering.SizedGroupError as ex: # error inside sized group # processOneIter already flushed group so do not flush stream - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser sized group error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser sized group error: %s", ex.args[0]) except (kering.ColdStartError, kering.ExtractionError) as ex: # some extraction error - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Parser msg extraction error: %s", ex.args[0]) else: - logger.error("Parser msg extraction error: %s\n", ex.args[0]) + logger.error("Parser msg extraction error: %s", ex.args[0]) del ims[:] # delete rest of stream to force cold restart except (kering.ValidationError, Exception) as ex: # non Extraction Error # Non extraction errors happen after successfully extracted from stream # so we don't flush rest of stream just resume - if logger.isEnabledFor(logging.ERROR): - logger.exception("Parser msg non-extraction error: %s\n", ex.args[0]) - else: - logger.error("Parser msg non-extraction error: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.TRACE): + logger.exception("Parser msg non-extraction error: %s", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.error("Parser msg non-extraction error: %s", ex.args[0]) yield return True # should never return def msgParsator(self, ims=None, framed=True, pipeline=False, - kvy=None, tvy=None, exc=None, rvy=None, vry=None): + kvy=None, tvy=None, exc=None, rvy=None, vry=None, + local=None, gvrsn=Vrsn_1_0): """ Returns generator that upon each iteration extracts and parses msg with attached crypto material (signature etc) from incoming message @@ -684,6 +670,10 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, exc (Exchanger) route EXN message types to this instance rvy (Revery): reply (RPY) message handler vry (Verifier) ACDC credential processor + local (bool): True means event source is local (protected) for validation + False means event source is remote (unprotected) for validation + None means use default .local + gvrsn (Versionage): instance of genera version of CESR code tables Logic: Currently only support couters on attachments not on combined or @@ -702,6 +692,9 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, """ + local = local if local is not None else self.local + local = True if local else False + serdery = serdering.Serdery(version=kering.Version) if ims is None: @@ -710,7 +703,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, while not ims: yield - cold = self.sniff(ims) # check for spurious counters at front of stream + cold = sniff(ims) # check for spurious counters at front of stream if cold in (Colds.txt, Colds.bny): # not message error out to flush stream # replace with pipelining here once CESR message format supported. raise kering.ColdStartError("Expecting message counter tritet={}" @@ -725,16 +718,6 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, else: # extracted and stripped successfully break # break out of while loop - - #while True: # extract and deserialize message from ims - #try: - #sadder = Sadder(raw=ims) - #except kering.ShortageError as ex: # need more bytes - #yield - #else: # extracted successfully - #del ims[:sadder.size] # strip off event from front of ims - #break - sigers = [] # list of Siger instances of attached indexed controller signatures wigers = [] # list of Siger instance of attached indexed witness signatures cigars = [] # List of cigars to hold nontrans rct couplets @@ -755,16 +738,17 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # List of tuples from extracted SAD path sig groups from non-trans identifiers sadcigs = [] # each converted group is path plus list of non-trans sigs pathed = [] # grouped attachments targetting a subpath + essrs = [] # group texter pipelined = False # all attachments in one big pipeline counted group # extract and deserialize attachments try: # catch errors here to flush only counted part of stream # extract attachments must start with counter so know if txt or bny. while not ims: yield - cold = self.sniff(ims) # expect counter at front of attachments + cold = sniff(ims) # expect counter at front of attachments if cold != Colds.msg: # not new message so process attachments ctr = yield from self._extractor(ims=ims, klas=Counter, cold=cold) - if ctr.code == CtrDex.AttachedMaterialQuadlets: # pipeline ctr? + if ctr.code == CtrDex_1_0.AttachmentGroup: # pipeline ctr? pipelined = True # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 @@ -786,7 +770,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # iteratively process attachment counters (all non pipelined) while True: # do while already extracted first counter is ctr - if ctr.code == CtrDex.ControllerIdxSigs: + if ctr.code == CtrDex_1_0.ControllerIdxSigs: for i in range(ctr.count): # extract each attached signature siger = yield from self._extractor(ims=ims, klas=Siger, @@ -794,7 +778,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) sigers.append(siger) - elif ctr.code == CtrDex.WitnessIdxSigs: + elif ctr.code == CtrDex_1_0.WitnessIdxSigs: for i in range(ctr.count): # extract each attached signature wiger = yield from self._extractor(ims=ims, klas=Siger, @@ -802,7 +786,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) wigers.append(wiger) - elif ctr.code == CtrDex.NonTransReceiptCouples: + elif ctr.code == CtrDex_1_0.NonTransReceiptCouples: # extract attached rct couplets into list of sigvers # verfer property of cigar is the identifier prefix # cigar itself has the attached signature @@ -812,7 +796,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, pipelined=pipelined): cigars.append(cigar) - elif ctr.code == CtrDex.TransReceiptQuadruples: + elif ctr.code == CtrDex_1_0.TransReceiptQuadruples: # extract attaced trans receipt vrc quadruple # spre+ssnu+sdig+sig # spre is pre of signer of vrc @@ -839,7 +823,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) trqs.append((prefixer, seqner, saider, siger)) - elif ctr.code == CtrDex.TransIdxSigGroups: + elif ctr.code == CtrDex_1_0.TransIdxSigGroups: # extract attaced trans indexed sig groups each made of # triple pre+snu+dig plus indexed sig group # pre is pre of signer (endorser) of msg @@ -852,7 +836,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, pipelined=pipelined): tsgs.append((prefixer, seqner, saider, isigers)) - elif ctr.code == CtrDex.TransLastIdxSigGroups: + elif ctr.code == CtrDex_1_0.TransLastIdxSigGroups: # extract attaced signer seal indexed sig groups each made of # identifier pre plus indexed sig group # pre is pre of signer (endorser) of msg @@ -867,10 +851,10 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, klas=Counter, cold=cold, abort=pipelined) - if ictr.code != CtrDex.ControllerIdxSigs: + if ictr.code != CtrDex_1_0.ControllerIdxSigs: raise kering.UnexpectedCountCodeError("Wrong " "count code={}.Expected code={}." - "".format(ictr.code, CtrDex.ControllerIdxSigs)) + "".format(ictr.code, CtrDex_1_0.ControllerIdxSigs)) isigers = [] for i in range(ictr.count): # extract each attached signature isiger = yield from self._extractor(ims=ims, @@ -880,7 +864,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, isigers.append(isiger) ssgs.append((prefixer, isigers)) - elif ctr.code == CtrDex.FirstSeenReplayCouples: + elif ctr.code == CtrDex_1_0.FirstSeenReplayCouples: # extract attached first seen replay couples # snu+dtm # snu is fn (first seen ordinal) of event @@ -896,7 +880,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) frcs.append((firner, dater)) - elif ctr.code == CtrDex.SealSourceCouples: + elif ctr.code == CtrDex_1_0.SealSourceCouples: # extract attached first seen replay couples # snu+dig # snu is sequence number of event @@ -912,7 +896,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) sscs.append((seqner, saider)) - elif ctr.code == CtrDex.SealSourceTriples: + elif ctr.code == CtrDex_1_0.SealSourceTriples: # extract attached anchoring source event information # pre+snu+dig # pre is prefix of event @@ -933,7 +917,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, abort=pipelined) ssts.append((prefixer, seqner, saider)) - elif ctr.code == CtrDex.SadPathSigGroup: + elif ctr.code == CtrDex_1_0.SadPathSigGroups: path = yield from self._extractor(ims, klas=Pather, cold=cold, @@ -948,22 +932,22 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, root=path, cold=cold, pipelined=pipelined): - if code == CtrDex.TransIdxSigGroups: + if code == CtrDex_1_0.TransIdxSigGroups: sadtsgs.append(sigs) else: sadcigs.append(sigs) - elif ctr.code == CtrDex.SadPathSig: - for code, sigs in self._sadPathSigGroup(ctr=ctr, - ims=ims, - cold=cold, - pipelined=pipelined): - if code == CtrDex.TransIdxSigGroups: - sadtsgs.append(sigs) - else: - sadcigs.append(sigs) + elif ctr.code == CtrDex_1_0.PathedMaterialGroup: # pathed ctr? + # compute pipelined attached group size based on txt or bny + pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 + while len(ims) < pags: # wait until rx full pipelned group + yield - elif ctr.code == CtrDex.PathedMaterialQuadlets: # pathed ctr? + pims = ims[:pags] # copy out substream pipeline group + del ims[:pags] # strip off from ims + pathed.append(pims) + + elif ctr.code == CtrDex_1_0.BigPathedMaterialGroup: # pathed ctr? # compute pipelined attached group size based on txt or bny pags = ctr.count * 4 if cold == Colds.txt else ctr.count * 3 while len(ims) < pags: # wait until rx full pipelned group @@ -973,6 +957,15 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, del ims[:pags] # strip off from ims pathed.append(pims) + elif ctr.code == CtrDex_1_0.ESSRPayloadGroup: + for i in range(ctr.count): + texter = yield from self._extractor(ims, + klas=Matter, + cold=cold, + abort=pipelined) + essrs.append(texter) + + else: raise kering.UnexpectedCountCodeError("Unsupported count" " code={}.".format(ctr.code)) @@ -986,7 +979,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # group may switch stream state txt or bny if not ims: # end of frame break - cold = self.sniff(ims) + cold = sniff(ims) if cold == Colds.msg: # new message so attachments done break # finished attachments since new message else: # process until next message @@ -994,7 +987,7 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, # group may switch stream state txt or bny while not ims: yield # no frame so must wait for next message - cold = self.sniff(ims) # ctr or msg + cold = sniff(ims) # ctr or msg if cold == Colds.msg: # new message break # finished attachments since new message @@ -1009,18 +1002,15 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, if isinstance(serder, serdering.SerderKERI): ilk = serder.ilk # dispatch abased on ilk - #if sadder.proto == Protos.keri: - #serder = Serder(sad=sadder) - - #ilk = serder.ked["t"] # dispatch abased on ilk - if ilk in [Ilks.icp, Ilks.rot, Ilks.ixn, Ilks.dip, Ilks.drt]: # event msg firner, dater = frcs[-1] if frcs else (None, None) # use last one if more than one # when present assumes this is source seal of delegating event in delegator's KEL delseqner, delsaider = sscs[-1] if sscs else (None, None) # use last one if more than one if not sigers: - logger.debug("Parser: Missing attached signature(s) for evt = \n%s\n", serder.ked) - raise kering.ValidationError(f"Missing attached signature(s) for evt={serder.ked['d']}") + msg = f"Missing attached signature(s) for evt = {serder.ked['d']}" + logger.info(msg) + logger.debug("Event Body = \n%s\n", serder.pretty()) + raise kering.ValidationError(msg) try: kvy.processEvent(serder=serder, sigers=sigers, @@ -1028,44 +1018,56 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, delseqner=delseqner, delsaider=delsaider, firner=firner, - dater=dater) + dater=dater, + local=local) if cigars: - kvy.processReceiptCouples(serder, cigars, firner=firner) + kvy.processAttachedReceiptCouples(serder, cigars, + firner=firner, local=local) if trqs: - kvy.processReceiptQuadruples(serder, trqs, firner=firner) + kvy.processAttachedReceiptQuadruples(serder, trqs, + firner=firner, local=local) except AttributeError as ex: - logger.debug("Parser: No kevery to process so dropped msg = %s", serder.pretty()) - raise kering.ValidationError(f"No kevery to process so dropped msg={serder.ked['d']}") from ex + msg = f"No kevery to process so dropped msg={serder.said}" + logger.info(msg) + logger.debug("Event Body = \n%s\n", serder.pretty()) + raise kering.ValidationError(msg) from ex elif ilk in [Ilks.rct]: # event receipt msg (nontransferable) if not (cigars or wigers or tsgs): - logger.debug("Parser: Missing attached signatures on receipt msg event =\n%s\n", serder.pretty()) - raise kering.ValidationError(f"Missing attached sigs on receipt msg={serder.ked['d']}") + msg = f"Missing attached signatures on receipt msg sn={serder.sn} SAID={serder.said}" + logger.info(msg) + logger.debug("Receipt body=\n%s\n", serder.pretty()) + raise kering.ValidationError(msg) try: if cigars: - kvy.processReceipt(serder=serder, cigars=cigars) + kvy.processReceipt(serder=serder, cigars=cigars, + local=local) if wigers: - kvy.processReceiptWitness(serder=serder, wigers=wigers) + kvy.processReceiptWitness(serder=serder, wigers=wigers, + local=local) if tsgs: - kvy.processReceiptTrans(serder=serder, tsgs=tsgs) + kvy.processReceiptTrans(serder=serder, tsgs=tsgs, + local=local) except AttributeError: - msg = f"No kevery to process so dropped msg = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("No kevery to process so dropped msg" + "= {}.".format(serder.pretty())) + except kering.UnverifiedReplyError as e: + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Error processing reply = %s", e) + logger.debug("Reply Body=\n%s\n", serder.pretty()) + else: + logger.error("Error processing reply = %s", e) elif ilk in (Ilks.rpy,): # reply message if not (cigars or tsgs): - msg = f"Missing attached endorser signature(s) to reply msg = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("Missing attached endorser signature(s) " + "to reply msg = {}.".format(serder.pretty())) try: if cigars: # process separately so do not clash on errors @@ -1075,16 +1077,8 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, rvy.processReply(serder, tsgs=tsgs) # trans except AttributeError as e: - msg = f"No kevery to process so dropped msg = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) - except kering.UnverifiedReplyError as e: - if logger.isEnabledFor(logging.DEBUG): - logger.exception("Error processing reply = %s", e) - logger.debug("Reply Body=\n%s\n", serder.pretty()) - else: - logger.error("Error processing reply = %s", e) + raise kering.ValidationError("No revery to process so dropped msg" + "= {}.".format(serder.pretty())) from e elif ilk in (Ilks.qry,): # query message args = dict(serder=serder) @@ -1097,47 +1091,42 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, args["cigars"] = cigars else: - msg = f"Missing attached requester signature(s) to key log query msg = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("Missing attached requester signature(s) " + "to key log query msg = {}.".format(serder.pretty())) route = serder.ked["r"] if route in ["logs", "ksn", "mbx"]: try: kvy.processQuery(**args) - except AttributeError: - msg = f"No kevery to process so dropped msg = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) - except kering.QueryNotFoundError as e: # catch escrow error and log it - if logger.isEnabledFor(logging.TRACE): - logger.exception("Error processing query = %s", e) - logger.trace("Query Body=\n%s\n", serder.pretty()) - else: - logger.error("Error processing query = %s", e) + except AttributeError as e: + raise kering.ValidationError("No kevery to process so dropped msg" + "= {} from e = {}".format(serder.pretty(), e)) elif route in ["tels", "tsn"]: try: tvy.processQuery(**args) except AttributeError as e: - msg = f"No tevery to process so dropped msg = {serder.said} from {e}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("No tevery to process so dropped msg" + "= {} from {}.".format(serder.pretty(), e)) + except kering.QueryNotFoundError as e: # catch escrow error and log it + if logger.isEnabledFor(logging.TRACE): + logger.exception("Error processing query = %s", e) + logger.trace("Query Body=\n%s\n", serder.pretty()) + else: + logger.error("Error processing query = %s", e) else: - msg = f"Invalid resource type {route} so dropped msg = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("Invalid resource type {} so dropped msg" + "= {}.".format(route, serder.pretty())) elif ilk in (Ilks.exn,): args = dict(serder=serder) if pathed: args["pathed"] = pathed + if essrs: + args["essrs"] = essrs + try: if cigars: exc.processEvent(cigars=cigars, **args) @@ -1146,10 +1135,8 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, exc.processEvent(tsgs=tsgs, **args) except AttributeError: - msg = "No Exchange to process so dropped msg = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("No Exchange to process so dropped msg" + "= {}.".format(serder.pretty())) elif ilk in (Ilks.vcp, Ilks.vrt, Ilks.iss, Ilks.rev, Ilks.bis, Ilks.brv): # TEL msg @@ -1159,15 +1146,11 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, tvy.processEvent(serder=serder, seqner=seqner, saider=saider, wigers=wigers) except AttributeError as e: - msg = f"No Tevery to process so dropped msg = {serder.said}" - logger.debug(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("No tevery to process so dropped msg" + "= {}.".format(serder.pretty())) else: - msg = f"Unexpected message ilk = {ilk} for evt = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("Unexpected message ilk = {} for evt =" + " {}.".format(ilk, serder.pretty())) elif isinstance(serder, serdering.SerderACDC): ilk = serder.ilk # dispatch based on ilk @@ -1177,20 +1160,14 @@ def msgParsator(self, ims=None, framed=True, pipeline=False, prefixer, seqner, saider = ssts[-1] if ssts else (None, None, None) # use last one if more than one vry.processCredential(creder=serder, prefixer=prefixer, seqner=seqner, saider=saider) except AttributeError as e: - msg = f"No verifier to process so dropped credential {serder.said}" - logger.debug(msg) - logger.debug("Credential body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("No verifier to process so dropped credential" + "= {}.".format(serder.pretty())) else: - msg = f"Unexpected message ilk = {ilk} for evt = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("Unexpected message ilk = {} for evt =" + " {}.".format(ilk, serder.pretty())) else: - msg = f"Unexpected protocol type = {serder.proto} for event message = {serder.said}" - logger.info(msg) - logger.debug("Event body=\n%s\n", serder.pretty()) - raise kering.ValidationError(msg) + raise kering.ValidationError("Unexpected protocol type = {} for event message =" + " {}.".format(serder.proto, serder.pretty())) return True # done state diff --git a/src/keri/core/routing.py b/src/keri/core/routing.py index 686929684..29c87a9e3 100644 --- a/src/keri/core/routing.py +++ b/src/keri/core/routing.py @@ -183,10 +183,10 @@ def processReply(self, serder, cigars=None, tsgs=None): Escrow process logic is route dependent and is dispatched by route, i.e. route is address of buffer with route specific handler of escrow. """ - for k in kering.RPY_LABELS: - if k not in serder.ked: - raise kering.ValidationError(f"Missing element={k} from {coring.Ilks.rpy}" - f" msg={serder.ked}.") + #for k in eventing.RPY_LABELS: + #if k not in serder.ked: + #raise kering.ValidationError(f"Missing element={k} from {coring.Ilks.rpy}" + #f" msg={serder.ked}.") # fetch from serder to process ked = serder.ked @@ -261,32 +261,35 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, for cigar in cigars: # process each couple to verify sig and write to db if cigar.verfer.transferable: # ignore invalid transferable verfers + logger.info("Revery: skipped invalid transferable verfers " + "on reply said = %s", serder.said) continue # skip invalid transferable if not self.lax and cigar.verfer.qb64 in self.prefixes: # own cig if not self.local: # own cig when not local so ignore - logger.info("Kevery: skipped own attachment for AID %s" + logger.info("Revery: skipped own attachment for AID %s" " on non-local reply at route = %s", aid, serder.ked['r']) logger.debug("Reply Body=\n%s\n", serder.pretty()) + continue # skip own cig attachment on non-local reply msg if aid != cigar.verfer.qb64: # cig not by aid - logger.info("Kevery: skipped cig not from aid=" + logger.info("Revery: skipped cig not from aid=" "%s on reply at route %s", aid, serder.ked['r']) logger.debug("Reply Body=\n%s\n", serder.pretty()) continue # skip invalid cig's verfer is not aid if odater: # get old compare datetimes to see if later if dater.datetime <= odater.datetime: - logger.trace("Kevery: skipped stale update from " - "%s of reply at route= %s", aid, serder.ked['r']) + logger.trace("Revery: skipped stale update from " + "%s of reply at route= %s", aid, serder.ked['r']) logger.trace("Reply Body=\n%s\n", serder.pretty()) continue # skip if not later # raise ValidationError(f"Stale update of {route} from {aid} " # f"via {Ilks.rpy}={serder.ked}.") if not cigar.verfer.verify(cigar.raw, serder.raw): # cig not verify - logger.info("Kevery: skipped non-verifying cig from " + logger.info("Revery: skipped non-verifying cig from " "%s on reply at route = %s", cigar.verfer.qb64, serder.ked['r']) logger.debug("Reply Body=\n%s\n", serder.pretty()) continue # skip if cig not verify @@ -300,14 +303,16 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, for prefixer, seqner, ssaider, sigers in tsgs: # iterate over each tsg if not self.lax and prefixer.qb64 in self.prefixes: # own sig if not self.local: # own sig when not local so ignore - logger.info("Kevery process: skipped own attachment" - " on nonlocal reply msg=\n%s\n", serder.pretty()) + logger.debug("Revery: skipped own attachment " + "on nonlocal reply said=%s", serder.said) + logger.debug("event=\n%s\n", serder.pretty()) continue # skip own sig attachment on non-local reply msg spre = prefixer.qb64 if aid != spre: # sig not by aid - logger.info("Kevery process: skipped signature not from aid=" - "%s on reply msg=\n%s\n", aid, serder.pretty()) + logger.debug("Revery: skipped signature not from aid = " + "%s on reply said=%s", aid, serder.said) + logger.debug("event=\n%s\n", serder.pretty()) continue # skip invalid signature is not from aid if osaider: # check if later logic sn > or sn == and dt > @@ -315,18 +320,18 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, _, osqr, _, _ = otsgs[0] # zeroth should be authoritative if seqner.sn < osqr.sn: # sn earlier - logger.info("Kevery process: skipped stale key state sig" - "from %s sn=%s<%s on reply msg=\n%s\n", - aid, seqner.sn, osqr.sn, serder.pretty()) + logger.info("Revery: skipped stale key state sig " + "from %s sn=%s<%s on reply said=%s", + aid, seqner.sn, osqr.sn, serder.said) + logger.debug("event=\n%s\n", serder.pretty()) continue # skip if sn earlier if seqner.sn == osqr.sn: # sn same so check datetime if odater: if dater.datetime <= odater.datetime: - logger.debug("Kevery process: skipped stale key " - "state sig datetime from %s on reply msg = %s", - aid, serder.said) - logger.debug("Reply Body=\n%s\n", serder.pretty()) + logger.info("Revery: skipped stale key state sig datetime " + "from %s on reply said=%s", aid, serder.said) + logger.debug("event=\n%s\n", serder.pretty()) continue # skip if not later # retrieve sdig of last event at sn of signer. @@ -334,6 +339,8 @@ def acceptReply(self, serder, saider, route, aid, osaider=None, if sdig is None: # create cue here to request key state for sprefixer signer # signer's est event not yet in signer's KEL + logger.info("Revery: escrowing without key state for signer" + " on reply said=%s", serder.said) self.escrowReply(serder=serder, saider=saider, dater=dater, route=route, prefixer=prefixer, seqner=seqner, ssaider=ssaider, sigers=sigers) @@ -462,7 +469,7 @@ def processEscrowReply(self): quadruple (prefixer, seqner, diger, siger) """ - for (route, ion), saider in self.db.rpes.getIoItemIter(): + for (route,), saider in self.db.rpes.getItemIter(): try: tsgs = eventing.fetchTsgs(db=self.db.ssgs, saider=saider) @@ -479,7 +486,7 @@ def processEscrowReply(self): datetime.timedelta(seconds=self.TimeoutRPE)): # escrow stale so raise ValidationError which unescrows below logger.info("Kevery unescrow error: Stale reply escrow " - " at route = %s\n", route) + " at route = %s", route) raise kering.ValidationError(f"Stale reply escrow at route = {route}.") @@ -489,28 +496,27 @@ def processEscrowReply(self): # still waiting on missing prior event to validate if logger.isEnabledFor(logging.TRACE): logger.trace("Kevery unescrow attempt failed: %s\n", ex.args[0]) - logger.exception("Kevery unescrow attempt failed: %s\n", ex.args[0]) except Exception as ex: # other error so remove from reply escrow self.db.rpes.rem(keys=(route, ), val=saider) # remove escrow only self.removeReply(saider) # remove escrow reply artifacts if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.error("Kevery unescrowed due to error: %s", ex.args[0]) else: # unescrow succeded self.db.rpes.rem(keys=(route, ), val=saider) # remove escrow only - logger.info("Kevery unescrow succeeded for reply = %s", serder.said) - logger.debug("Reply Body=\n%s\n", serder.pretty()) + logger.info("Revery unescrow succeeded for reply said=%s", serder.said) + logger.debug("event=\n%s\n", serder.pretty()) except Exception as ex: # log diagnostics errors etc self.db.rpes.rem(keys=(route,), val=saider) # remove escrow only self.removeReply(saider) # remove escrow reply artifacts if logger.isEnabledFor(logging.DEBUG): - logger.exception("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.exception("Kevery unescrowed due to error: %s", ex.args[0]) else: - logger.error("Kevery unescrowed due to error: %s\n", ex.args[0]) + logger.error("Kevery unescrowed due to error: %s", ex.args[0]) class Route: diff --git a/src/keri/core/scheming.py b/src/keri/core/scheming.py index 5c87667a2..203ec9500 100644 --- a/src/keri/core/scheming.py +++ b/src/keri/core/scheming.py @@ -12,7 +12,7 @@ import msgpack from . import coring -from .coring import MtrDex, Serials, Saider, Saids +from .coring import MtrDex, Kinds, Saider, Saids from .. import help, kering from ..kering import ValidationError, DeserializeError @@ -109,7 +109,7 @@ def resolve(self, uri): return self.resolver.resolve(uri) - def load(self, raw, kind=Serials.json): + def load(self, raw, kind=Kinds.json): """ Schema loader Loads schema based on kind by performing deserialization on raw bytes of schema @@ -122,21 +122,21 @@ def load(self, raw, kind=Serials.json): tuple: (dict, Serials, Saider) of schema """ - if kind == Serials.json: + if kind == Kinds.json: try: sed = json.loads(raw.decode("utf-8")) except Exception as ex: raise DeserializeError("Error deserializing JSON: {} {}" "".format(raw.decode("utf-8"), ex)) - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: try: sed = msgpack.loads(raw) except Exception as ex: raise DeserializeError("Error deserializing MGPK: {} {}" "".format(raw, ex)) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: try: sed = cbor.loads(raw) except Exception as ex: @@ -158,7 +158,7 @@ def load(self, raw, kind=Serials.json): return sed, kind, saider @staticmethod - def dump(sed, kind=Serials.json): + def dump(sed, kind=Kinds.json): """ Serailize schema based on kind Parameters: @@ -266,7 +266,8 @@ class Schemer: """ - def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.Blake3_256): + def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), + code=MtrDex.Blake3_256, verify=True): """ Initialize instance of Schemer Deserialize if raw provided @@ -274,14 +275,19 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B When serializing if kind provided then use kind instead of field in sed Parameters: - raw (bytes): of serialized schema - sed (dict): dict or None - if None its deserialized from raw - typ (JSONSchema): type of schema - kind (serialization): kind string value or None (see namedtuple coring.Serials) - supported kinds are 'json', 'cbor', 'msgpack', 'binary' - if kind (None): then its extracted from ked or raw - code (MtrDex): default digest code + raw (bytes): of serialized schema + sed (dict): dict or None + if None its deserialized from raw + typ (JSONSchema): type of schema + kind (serialization): kind string value or None (see namedtuple coring.Serials) + supported kinds are 'json', 'cbor', 'msgpack', 'binary' + if kind (None): then its extracted from ked or raw + code (MtrDex): default digest code + verify (bool): True means verify said(s) of given raw or sad. + Raises ValidationError if verification fails + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. """ @@ -295,7 +301,7 @@ def __init__(self, raw=b'', sed=None, kind=None, typ=JSONSchema(), code=MtrDex.B else: raise ValueError("Improper initialization need raw or sed.") - if not self._verify_schema(): + if verify and not self._verify_schema(): raise ValidationError("invalid kind {} for schema {}" "".format(self.kind, self.sed)) diff --git a/src/keri/core/serdering.py b/src/keri/core/serdering.py index 92db8221a..a56eb1613 100644 --- a/src/keri/core/serdering.py +++ b/src/keri/core/serdering.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- """ keri.core.serdering module @@ -6,71 +6,204 @@ import copy import json from collections import namedtuple +from collections.abc import Mapping +from dataclasses import dataclass, asdict, field import cbor2 as cbor import msgpack import pysodium import blake3 import hashlib +from ordered_set import OrderedSet as oset + from .. import kering -from ..kering import (ValidationError, MissingFieldError, +from ..kering import (ValidationError, MissingFieldError, ExtraFieldError, + AlternateFieldError, InvalidValueError, ShortageError, VersionError, ProtocolError, KindError, DeserializeError, FieldError, SerializeError) -from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_1_1, - VERRAWSIZE, VERFMT, VERFULLSIZE) -from ..kering import Protos, Serials, Rever, versify, deversify, Ilks -from ..core import coring -from .coring import MtrDex, DigDex, PreDex, Saids, Digestage -from .coring import Matter, Saider, Verfer, Diger, Number, Tholder +from ..kering import (Versionage, Version, Vrsn_1_0, Vrsn_2_0, + VERRAWSIZE, VERFMT, + MAXVERFULLSPAN, VER1FULLSPAN, VER2FULLSPAN) +from ..kering import SMELLSIZE, Smellage, smell + +from ..kering import Protocols, Kinds, versify, deversify, Ilks from .. import help from ..help import helping + +from . import coring +from .coring import (MtrDex, DigDex, PreDex, NonTransDex, PreNonDigDex, + Saids, Digestage) +from .coring import (Matter, Saider, Verfer, Diger, Number, Tholder, Tagger, + Ilker, Traitor, Verser, ) + +from .counting import GenDex, Counter, Codens, SealDex_2_0 + +from .structing import Sealer, SClanDom + + + logger = help.ogler.getLogger() -""" -Fieldage - saids (dict): keyed by saidive field labels with values as default codes - alls (dict): keyed by all field labels including saidive ones - with values as default codes - -Example: - Fields = Labelage(saids={'d': DigDex.Blake3_256}, - alls={'v': '','d':''}) -""" -Fieldage = namedtuple("Fieldage", "saids alls") #values are dicts -""" -Reapage - proto (str): protocol type value of Protos examples 'KERI', 'ACDC' - major (str): single char hex string of major version number - minor (str): single char hex string of minor version number - kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' + +@dataclass +class FieldDom: + """ + Field configuration dataclass for Serder messages. Provides field labels + and default field values for a given ilk (message type). + + Attributes: + alls (dict): Allowed fields (not extra) + alls must not be empty since at least version string + or protocol version filed is always required. + Fields in alls that appear must appear in order. + + opts (dict): Optional fields within alls. + opts defaults to empty. + When opts is empty than all alls are required. + Any fields in alls but not in opts are required. + opts is a subset of alls + + alts (dict): Alternate fields within alls. + alts defaults to empty. + An alt field means that one or the other of two fields is + allowed but not both. Two entries in alts are required, one + for each field in an alt pair. the alt pair key value are + the two labels. One entry for each order. + all alts must be in opts since both can't be required. + Suppose 'a' and 'A' are an alternate pair then alts = + { 'a': 'A', 'A': 'a'}. This allows the presence of one to + block the presence of the other by looking up the one present + as key to see the value of the one to block. + + saids (dict): are saidive fields whose value may be computed as a said of the message. + saids defaults to empty + when provided a field in saids indicates the field value is saidive. + A simple SAID field value is always computed. + An AID SAID field value is only computed when its code indicates. + saids is a subset of alls + + strict (bool): determines if alls is strict, no extra fields are allowed + strict defaults to True. + True means no extra fields are allowed, only those in alls. + False means extra fields are allowed besides thos in alls. + Extra fields are fields not in alls. + Extra fields may appear in any order after the last field + in alls. + + + When strict: no extras allowed + Any fields not in alls raise error + If opts is empty then all alls are required in order + If opts is not empty then fields in opts are optional but the rest of + the fields in alls are required + + When not strict: extras allowed + Any fields not in alls must appear after all fields in alls + If opts is empty then all alls are required in order + If opts is not empty then fields in opts are optional but the rest of + the fields in alls are required + + """ + alls: dict # all allowed fields when strict + opts: dict = field(default_factory=dict) # optional fields + alts: dict = field(default_factory=dict) # alternate optional fields + saids: dict = field(default_factory=dict) # saidive fields + strict: bool = True # only alls allowed no extras + + def __iter__(self): + return iter(asdict(self)) + +"""Design Notes: + +Problems is that sniff only determines if counter not which type of +counter. Then smell does regex lookahead to find out which serialization +when not count code but extractor does not look ahead but strips from +stream. So when possibility that CESR message is next either need to +not strip from stream when extracting or if counter is message +then grab rest of frame and reattach so raw in Serder includes the +message counter. Latter is better since always keep counter around +until later. So need to check counter type and if message then +extract rest of counter frame (message) and reattach counter raw. +Then can call Serder with raw and smellage that indicates CESR kind + +But this does not solve the problem of using the Serder subclass +for the given protocol. Merely knowing is a CESR message is not +enough also have to know the protocol which comes in the version +field (not version string). + +One solution is to modify smell so that it also can lookahead and +see the version field. Or lookahead and see the version field with +count codes in front. Problem is that the Regexes don't separate +cleanly. + +Another solution is to use distinct function for cesr native called +snuff like smell but regex only for CESR native. Reap can be told which +because sniff tells which it is. +So question for snuff is should it be searching over the counter or should it +start at version field. This changes regex so forced start at front of raw. +so if reattach counter but use skip then can snuff at start of string. +Begine regex with b'^' or b'\A' to match at start of string. + +So change Smellage to return extra field that has gvrsn when used by snuff +so can use Smellage for both smell and snuff in both reap and inhale +where smellage is used. Change egacy uses of smell to ignore extra value. + + + +Lets try that as it works the best. + +while True: # extract, deserialize, and strip message from ims + try: + serder = serdery.reap(ims=ims) # can set version here + except kering.ShortageError as ex: # need more bytes + yield + else: # extracted and stripped successfully + break # break out of while loop + +ctr = yield from self._extractor(ims=ims, klas=Counter, cold=cold) +if ctr.code == CtrDex.AttachmentGroup: # pipeline ctr? + pipelined = True + +@staticmethod +def _extractor(ims, klas, cold=Colds.txt, abort=False): + while True: + try: + if cold == Colds.txt: + return klas(qb64b=ims, strip=True) + elif cold == Colds.bny: + return klas(qb2=ims, strip=True) + else: + raise kering.ColdStartError("Invalid stream state cold={}.".format(cold)) + except kering.ShortageError as ex: + if abort: # pipelined pre-collects full frame before extracting + raise # bad pipelined frame so abort by raising error + yield + + """ -Reapage = namedtuple("Reapage", "proto major minor kind size") class Serdery: """Serder factory class for generating serder instances by protocol type from an incoming message stream. - - """ - def __init__(self, *, version=None): + def __init__(self, *pa, **kwa): """Init instance Parameters: - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version + """ - self.version = version # default version + pass - def reap(self, ims, *, version=None): + def reap(self, ims, genus=GenDex.KERI, gvrsn=Vrsn_2_0, native=False, skip=0): """Extract and return Serder subclass based on protocol type reaped from version string inside serialized raw of Serder. @@ -81,27 +214,34 @@ def reap(self, ims, *, version=None): Parameters: ims (bytearray) of serialized incoming message stream. Assumes start of stream is raw Serder. - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version - """ - version = version if version is not None else self.version - - if len(ims) < Serder.InhaleSize: - raise ShortageError(f"Need more raw bytes for Serdery to reap.") + genus (str): CESR genus code from stream parser. + Provides genus of enclosing stream top-level or nested group + gvrsn (Versionage): instance CESR genus code table version (Major, Minor) + Provides genus of enclosing stream top-level or nested group + native (bool): True means sniff determined may be CESR native message + so snuff instead of smell. + False means sniff determined not CESR native i.e + JSON, CBOR, MGPK field map. so use smell. Default False + skip (int): bytes to skip at front of ims. Useful when CESR native + serialization where skip is size of the message counter so smell + does need to see counter + + """ + if native: + pass + #smellage = smell(memoryview(ims)[skip:]) # does not copy to skip - match = Rever.search(ims) # Rever regex takes bytes/bytearray not str - if not match or match.start() > Serder.MaxVSOffset: - raise VersionError(f"Invalid version string for Serder raw = " - f"{ims[: Serder.InhaleSize]}.") - - reaped = Reapage(*match.group("proto", "major", "minor", "kind", "size")) - - if reaped.proto == Protos.keri.encode("utf-8"): - return SerderKERI(raw=ims, strip=True, version=version, reaped=reaped) - elif reaped.proto == Protos.acdc.encode("utf-8"): - return SerderACDC(raw=ims, strip=True, version=version, reaped=reaped) else: - raise ProtocolError(f"Unsupported protocol type = {reaped.proto}.") + smellage = smell(ims) + + if smellage.proto == Protocols.keri: + return SerderKERI(raw=ims, strip=True, smellage=smellage, + genus=genus, gvrsn=gvrsn) + elif smellage.proto == Protocols.acdc: + return SerderACDC(raw=ims, strip=True, smellage=smellage, + genus=genus, gvrsn=gvrsn) + else: + raise ProtocolError(f"Unsupported protocol type = {smellage.proto}.") @@ -132,17 +272,32 @@ class Serder: generation and verification in addition to the required fields. Class Attributes: - MaxVSOffset (int): Maximum Version String Offset in bytes/chars - InhaleSize (int): Minimum raw buffer size needed to inhale - Labels (dict): Protocol specific dict of field labels keyed by ilk - (packet type string value). None is default key when no ilk needed. - Each entry is a + Dummy (str): dummy character for computing SAIDs + Spans (dict): version string spans keyed by version + Digests (dict): map of digestive codes. Should be same set of codes as + in coring.DigestCodex coring.DigDex so that .digestive property works. + Use unit tests to ensure codex sets match + Protocol (str): class specific message protocol + Proto (str): default message protocol + Vrsn (Versionage): default version + Kind (str): default serialization kind one of Serials + Fields (dict): nested dict of field labels keyed by protocol, version, + and message type (ilk). Felds labels are provided with a Fieldage + named tuple (saids, reqs, alls) that governs field type and presence. + None is default message type (ilk) when no ilk needed in a message. + See below for detailed logic associated with Fields class attribute Properties: raw (bytes): of serialized event only sad (dict): self addressed data dict + genus (str): CESR genus code + gvrsn (Versionage): instance CESR genus code table version (Major, Minor) proto (str): Protocolage value as protocol identifier such as KERI, ACDC - version (Versionage): protocol version (Major, Minor) + alias of .protocol + protocol (str): Protocolage value as protocol identifier such as KERI, ACDC + alias of .proto + vrsn (Versionage): protocol version (Major, Minor) alias of .version + version (Versionage): protocol version (Major, Minor) alias of .vrsn kind (str): serialization kind coring.Serials such as JSON, CBOR, MGPK, CESR size (int): number of bytes in serialization said (str): qb64 said of .raw given by appropriate field @@ -151,10 +306,11 @@ class Serder: Hidden Attributes: - ._raw is bytes of serialized event only - ._sad is key event dict + ._raw (bytes): serialized message + ._sad (dict): sad dict (key event dict) + ._cvrsn (Versionage): CESR code table version ._proto (str): Protocolage value as protocol type identifier - ._version is Versionage instance of event version + ._vrsn is Versionage instance of event version ._kind is serialization kind string value (see namedtuple coring.Serials) supported kinds are 'json', 'cbor', 'msgpack', 'binary' ._size is int of number of bytes in serialed event only @@ -178,180 +334,206 @@ class Serder: Note: loads and jumps of json use str whereas cbor and msgpack use bytes + + Fields: + Each element of Fields is a FieldDom dataclass instance with four attributes: + alls (dict): + opts (dict): + saids (dict): + strict (bool): + """ + Dummy = "#" # dummy spaceholder char for SAID. Must not be a valid Base64 char - MaxVSOffset = 12 - InhaleSize = MaxVSOffset + VERFULLSIZE # min buffer size to inhale - - Dummy = "#" # dummy spaceholder char for said. Must not be a valid Base64 char - - # should be same set of codes as in coring.DigestCodex coring.DigDex so - # .digestive property works. Use unit tests to ensure codex sets match - Digests = { - DigDex.Blake3_256: Digestage(klas=blake3.blake3, size=None, length=None), - DigDex.Blake2b_256: Digestage(klas=hashlib.blake2b, size=32, length=None), - DigDex.Blake2s_256: Digestage(klas=hashlib.blake2s, size=None, length=None), - DigDex.SHA3_256: Digestage(klas=hashlib.sha3_256, size=None, length=None), - DigDex.SHA2_256: Digestage(klas=hashlib.sha256, size=None, length=None), - DigDex.Blake3_512: Digestage(klas=blake3.blake3, size=None, length=64), - DigDex.Blake2b_512: Digestage(klas=hashlib.blake2b, size=None, length=None), - DigDex.SHA3_512: Digestage(klas=hashlib.sha3_512, size=None, length=None), - DigDex.SHA2_512: Digestage(klas=hashlib.sha512, size=None, length=None), - } + # Spans dict keyed by version (Versionage instance) of version string span (size) + Spans = {Vrsn_1_0: VER1FULLSPAN, Vrsn_2_0: VER2FULLSPAN} - #override in subclass to enforce specific protocol - Protocol = None # required protocol, None means any in Protos is ok + # map seal clan names to seal counter code for grouping seals in anchor list + ClanCodes = dict() + ClanCodes[SClanDom.SealDigest.__name__] = SealDex_2_0.DigestSealSingles + ClanCodes[SClanDom.SealRoot.__name__] = SealDex_2_0.MerkleRootSealSingles + ClanCodes[SClanDom.SealBacker.__name__] = SealDex_2_0.BackerRegistrarSealCouples + ClanCodes[SClanDom.SealLast.__name__] = SealDex_2_0.SealSourceLastSingles + ClanCodes[SClanDom.SealTrans.__name__] = SealDex_2_0.SealSourceCouples + ClanCodes[SClanDom.SealEvent.__name__] = SealDex_2_0.SealSourceTriples - Proto = Protos.keri # default protocol type + # map seal counter code to seal clan name for parsing seal groups in anchor list + CodeClans = { val: key for key, val in ClanCodes.items()} # invert dict + + #override in subclass to enforce specific protocol + Protocol = None # class based message protocol, None means any in Protocols is ok + Proto = Protocols.keri # default message protocol type for makify on base Serder Vrsn = Vrsn_1_0 # default protocol version for protocol type - Kind = Serials.json # default serialization kind + Kind = Kinds.json # default serialization kind + CVrsn = Vrsn_2_0 # default CESR code table version # Nested dict keyed by protocol. # Each protocol value is a dict keyed by ilk. # Each ilk value is a Labelage named tuple with saids, codes and fields # ilk value of None is default for protocols that support ilkless packets - Fields = { - Protos.keri: + Fields = \ + { + Protocols.keri: + { + Vrsn_1_0: { - Vrsn_1_0: - { - Ilks.icp: Fieldage(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[])), - Ilks.rot: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], a=[])), - Ilks.ixn: Fieldage({Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', a=[])), - Ilks.dip: Fieldage(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[], - di='')), - Ilks.drt: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], a=[])), - Ilks.rct: Fieldage(saids={}, - alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='', rr='', - q={})), - Ilks.rpy: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='',a=[])), - Ilks.pro: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='', rr='', - q={})), - Ilks.bar: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', dt='', r='',a=[])), - Ilks.exn: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='', d='', i="", rp='', p="", dt='', r='',q={}, - a=[], e={})), - Ilks.vcp: Fieldage(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', c=[], - bt='0', b=[], n='')), - Ilks.vrt: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', p='', s='0', - bt='0', br=[], ba=[])), - Ilks.iss: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - dt='')), - Ilks.rev: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - p='', dt='')), - Ilks.bis: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', ra={}, - dt='')), - Ilks.brv: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', p='', ra={}, - dt='')), - }, - Vrsn_1_1: - { - Ilks.icp: Fieldage(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[])), - Ilks.rot: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], c=[], a=[])), - Ilks.ixn: Fieldage({Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', a=[])), - Ilks.dip: Fieldage(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', kt='0', - k=[], nt='0', n=[], bt='0', b=[], c=[], a=[], - di='')), - Ilks.drt: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', s='0', p='', - kt='0',k=[], nt='0', n=[], bt='0', br=[], - ba=[], c=[], a=[])), - Ilks.rct: Fieldage(saids={}, - alls=dict(v='', t='',d='', i='', s='0')), - Ilks.qry: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='', rr='', - q={})), - Ilks.rpy: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='',a=[])), - Ilks.pro: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='', rr='', - q={})), - Ilks.bar: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='',d='', i='', dt='', r='',a=[])), - Ilks.exn: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', t='', d='', i="", rp="", p="", dt='', r='', q={}, - a=[], e={})), - }, + Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[]), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[], di=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), + Ilks.qry: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + rr='',q={}), + saids={Saids.d: DigDex.Blake3_256},), + Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.pro: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + rr='',q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.bar: FieldDom(alls=dict(v='', t='',d='', dt='', r='', + a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", rp="", + p="", dt='', r='',q={}, a=[], e={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.vcp: FieldDom(alls=dict(v='', t='',d='', i='', ii='', + s='0', c=[], bt='0', b=[], n=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.vrt: FieldDom(alls=dict(v='', t='',d='', i='', p='', + s='0', bt='0', br=[], ba=[]), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.iss: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + ri='', dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.rev: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + ri='', p='', dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.bis: FieldDom(alls=dict(v='', t='',d='', i='', ii='', + s='0', ra={}, dt=''), + saids={Saids.d: DigDex.Blake3_256,}), + Ilks.brv: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', ra={}, dt=''), + saids={Saids.d: DigDex.Blake3_256,}), }, - Protos.crel: + Vrsn_2_0: { - Vrsn_1_1: - { - Ilks.vcp: Fieldage(saids={Saids.d: DigDex.Blake3_256, - Saids.i: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', c=[], - bt='0', b=[], u='')), - Ilks.vrt: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', p='', s='0', - bt='0', br=[], ba=[])), - Ilks.iss: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - dt='')), - Ilks.rev: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', ri='', - p='', dt='')), - Ilks.bis: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', ii='', s='0', ra={}, - dt='')), - Ilks.brv: Fieldage(saids={Saids.d: DigDex.Blake3_256,}, - alls=dict(v='', t='',d='', i='', s='0', p='', ra={}, - dt='')), - }, + Ilks.icp: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[]), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.rot: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], c=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ixn: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.dip: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + kt='0', k=[], nt='0', n=[], bt='0', b=[], c=[], + a=[], di=''), + saids={Saids.d: DigDex.Blake3_256, + Saids.i: DigDex.Blake3_256,}), + Ilks.drt: FieldDom(alls=dict(v='', t='',d='', i='', s='0', + p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], + ba=[], c=[], a=[]), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rct: FieldDom(alls=dict(v='', t='',d='', i='', s='0')), + Ilks.qry: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='', rr='', q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rpy: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='',a={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.pro: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='', rr='', q={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.bar: FieldDom(alls=dict(v='', t='',d='', i='', dt='', + r='',a={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.xip: FieldDom(alls=dict(v='', t='', d='', i="", dt='', + r='', q={}, a={}), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.exn: FieldDom(alls=dict(v='', t='', d='', i="", x="", + p="", dt='', r='', q={}, a={}), + saids={Saids.d: DigDex.Blake3_256}), }, - Protos.acdc: + }, + Protocols.acdc: + { + Vrsn_1_0: { - Vrsn_1_0: - { - None: Fieldage(saids={Saids.d: DigDex.Blake3_256}, - alls=dict(v='', d='', i='', s='')), - } + None: FieldDom(alls=dict(v='', d='', u='', i='', + ri='', s='', a='', A='', e='', r=''), + opts=dict(u='', ri='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', + ri='', s='', a='', A='', e='', r=''), + opts=dict(u='', ri='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}, + strict=False), }, - } - - - # default ilk for each protocol at default version is zeroth ilk in dict - Ilks = dict() - for key, val in Fields.items(): - Ilks[key] = list(list(val.values())[0].keys())[0] + Vrsn_2_0: + { + None: FieldDom(alls=dict(v='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.acd: FieldDom(alls=dict(v='', t='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.ace: FieldDom(alls=dict(v='', t='', d='', u='', i='', + rd='', s='', a='', A='', e='', r=''), + opts=dict(u='', rd='', a='', A='', e='', r=''), + alts=dict(a="A", A="a"), + saids={Saids.d: DigDex.Blake3_256}, + strict=False), + Ilks.sch: FieldDom(alls=dict(v='', t='', d='', s=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.att: FieldDom(alls=dict(v='', t='', d='', a=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.agg: FieldDom(alls=dict(v='', t='', d='', A=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rul: FieldDom(alls=dict(v='', t='', d='', e=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.rip: FieldDom(alls=dict(v='', t='', d='', u='', i='', + s='', dt=''), + saids={Saids.d: DigDex.Blake3_256}), + Ilks.upd: FieldDom(alls=dict(v='', t='', d='', r='', s='', + p='', dt='', a=''), + saids={Saids.d: DigDex.Blake3_256}), + }, + }, + } - def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, - reaped=None, verify=True, makify=False, + def __init__(self, *, raw=b'', sad=None, strip=False, smellage=None, + genus=GenDex.KERI, gvrsn=Vrsn_2_0, verify=True, makify=False, proto=None, vrsn=None, kind=None, ilk=None, saids=None): """Deserialize raw if provided. Update properties from deserialized raw. Verifies said(s) embedded in sad as given by labels. @@ -363,24 +545,30 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, given by label(s) according to proto and ilk and code. Parameters: - raw (bytes): serialized event + raw (bytes | bytearray): serialized event sad (dict): serializable saidified field map of message. Ignored if raw provided strip (bool): True means strip (delete) raw from input stream bytearray after parsing. False means do not strip. Assumes that raw is bytearray when strip is True. - version (Versionage | None): instance supported protocol version - None means do not enforce a supported version - reaped (Reapage | None): instance of deconstructed version string - elements. If none or empty ignore otherwise assume that raw - already had its version string extracted (reaped) into the - elements of reaped. + smellage (Smellage | None): instance of deconstructed and converted + version string elements. If none or empty ignore otherwise assume + that raw already had its version string extracted (reaped) into the + elements of smellage. + genus (str): CESR genus str. Either provided by parser from stream + or generated by Serder to stream + gvrsn (Versionage): instance CESR genus code table version + Either provided by parser from stream genus version or desired when + generating Serder instance to stream verify (bool): True means verify said(s) of given raw or sad. - Raises ValidationError if verification fails - Ignore when raw not provided or when raw and saidify is True + False means don't verify. Useful to avoid unnecessary + reverification when deserializing from database + as opposed to over the wire reception. + Raises ValidationError if verification fails + Ignore when raw empty or when raw and saidify is True makify (bool): True means compute fields for sad including size and saids. - proto (str | None): desired protocol type str value of Protos + proto (str | None): desired protocol type str value of Protocols If None then its extracted from sad or uses default .Proto vrsn (Versionage | None): instance desired protocol version If None then its extracted from sad or uses default .Vrsn @@ -399,21 +587,16 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, """ + self._gvrsn = gvrsn + self._genus = genus if raw: # deserialize raw using property setter - # self._inhale works because it only references class attributes - sad, proto, vrsn, kind, size = self._inhale(raw=raw, - version=version, - reaped=reaped) - self._raw = bytes(raw[:size]) # crypto ops require bytes not bytearray - self._sad = sad - self._proto = proto - self._vrsn = vrsn - self._kind = kind - self._size = size + self._inhale(raw=raw, smellage=smellage) + # ._inhale updates ._raw, ._sad, ._proto, ._vrsn, ._kind, ._size + # primary said field label try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids.keys())[0] + label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] if label not in self._sad: raise FieldError(f"Missing primary said field in {self._sad}.") self._said = self._sad[label] # not verified @@ -426,7 +609,7 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, except TypeError: pass # ignore if bytes - if verify: # verify the said(s) provided in raw + if verify: # verify fields including the said(s) provided in raw try: self._verify() # raises exception when not verify except Exception as ex: @@ -437,40 +620,36 @@ def __init__(self, *, raw=b'', sad=None, strip=False, version=Version, elif sad or makify: # serialize sad into raw or make sad if makify: # recompute properties and said(s) and reset sad - # makify resets sad, raw, proto, version, kind, and size - self.makify(sad=sad, version=version, - proto=proto, vrsn=vrsn, kind=kind, ilk=ilk, saids=saids) + # makify resets sad, raw, proto, vrsn, kind, ilk, and size + self.makify(sad=sad, proto=proto, vrsn=vrsn, kind=kind, + ilk=ilk, saids=saids) + # .makify updates ._raw, ._sad, ._proto, ._vrsn, ._kind, ._size else: - # self._exhale works because it only access class attributes - raw, sad, proto, vrsn, kind, size = self._exhale(sad=sad, - version=version) - self._raw = raw - self._sad = sad - self._proto = proto - self._vrsn = vrsn - self._kind = kind - self._size = size - # primary said field label + self._exhale(sad=sad) + # .exhale updates ._raw, ._sad, ._proto, ._vrsn, ._kind, ._size + + # primary said field label + try: + label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids)[0] + if label not in self._sad: + raise DeserializeError(f"Missing primary said field in {self._sad}.") + self._said = self._sad[label] # not verified + except Exception: + self._said = None # no saidive field + + if verify: # verify fields including the said(s) provided in sad try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids.keys())[0] - if label not in self._sad: - raise DeserializeError(f"Missing primary said field in {self._sad}.") - self._said = self._sad[label] # not verified - except Exception: - self._said = None # no saidive field - - if verify: # verify the said(s) provided in sad - try: - self._verify() # raises exception when not verify - except Exception as ex: - logger.error("Invalid sad for Serder %s\n%s", - self.pretty(), ex.args[0]) - raise ValidationError(f"Invalid sad for Serder =" + self._verify() # raises exception when not verify + except Exception as ex: + logger.error("Invalid sad for Serder %s\n%s", + self.pretty(), ex.args[0]) + raise ValidationError(f"Invalid sad for Serder =" f"{self._sad}.") from ex else: - raise ValueError("Improper initialization need raw or sad or makify.") + raise InvalidValueError("Improper initialization need raw or sad " + f"or makify.") @@ -503,50 +682,86 @@ def _verify(self): Raises a ValidationError (or subclass) if any verification fails """ - if self.Protocol and self.proto != self.Protocol: - raise ValidationError(f"Expected protocol = {self.Protocol}, got " + if self.Protocol and self.proto != self.Protocol: # class required + raise ValidationError(f"Required protocol = {self.Protocol}, got " f"{self.proto} instead.") if self.proto not in self.Fields: raise ValidationError(f"Invalid protocol type = {self.proto}.") + if self.genus not in GenDex: # ensures self.genus != None + raise SerializeError(f"Invalid genus={self.genus}.") + + if getattr(GenDex, self.proto, None) != self.genus: + raise SerializeError(f"Incompatible protocol={self.proto} with " + f"genus={self.genus}.") + + if self.vrsn.major > self.gvrsn.major: + raise SerializeError(f"Incompatible major protocol version={self.vrsn}" + f" with major genus version={self.gvrsn}.") + + if self.vrsn not in self.Fields[self.proto]: + raise SerializeError(f"Invalid version={self.vrsn} for " + f"protocol={self.proto}.") + if self.ilk not in self.Fields[self.proto][self.vrsn]: raise ValidationError(f"Invalid packet type (ilk) = {self.ilk} for" f"protocol = {self.proto}.") + + fields = self.Fields[self.proto][self.vrsn][self.ilk] # get labelage - # ensure all required fields in alls are in sad - alls = fields.alls # dict of all field labels with default values - keys = list(self._sad) # get list of keys of self.sad - for key in list(keys): # make copy to mutate - if key not in alls: - del keys[keys.index(key)] # remove non required fields - - if list(alls.keys()) != keys: # forces ordering of labels in .sad - # special handling for 1.1.18 -> 1.1.32 - skip_missing_field = False - if self.ilk == Ilks.exn: - missing_keys = [key for key in alls.keys() if key not in keys] - if missing_keys: - if len(missing_keys) == 1 and 'rp' in missing_keys: - skip_missing_field = True - if skip_missing_field: - pass - else: - raise MissingFieldError(f"Missing one or more required fields from" - f"= {list(alls.keys())} in sad = " - f"{self._sad}.") + + alls = fields.alls # faster local reference + oalls = oset(alls) # ordereset of field labels + oopts = oset(fields.opts) # ordereset of field labels + oreqs = oalls - oopts # required fields + + oskeys = oset(self._sad) # ordered set of keys in sad (skeys) + osexts = oskeys - oalls # get ordered set of extras in sad (sexts) + if osexts and fields.strict: + raise ExtraFieldError(f"Unallowed extra field(s) = {list(osexts)} " + f"in sad.") + + osopts = oskeys - oreqs - osexts # subset of opts in sad + + osalls = oalls - (oopts - osopts) # subset of alls without missing opts in sad + + for k, v in fields.alts.items(): + if k in osopts and v in osopts: + raise AlternateFieldError(f"Unallowed, alternate fields '{k}' " + f"and '{v}' both present in sad.") + + # can't do set math osalls == oskeys - osexts becasue of osexts might be + # out-of-order so have to iterate to ensure osexts if any appear in oskeys + # after all fields in osalls + + # Special processing for compatibility with old KERIpy versions prior to 1.1.25 that do not send the 'rp' field in exn messages + skip_missing_fields = False + missing_keys = [key for key in osalls if key not in oskeys] # missing keys in sad + if missing_keys and self.ilk == Ilks.exn: # Skip missing fields only for Exn (Exchange) ilk + if len(missing_keys) == 1 and 'rp' in missing_keys: + skip_missing_fields = True + src_index = 0 + for i, label in enumerate(osalls): + # special missing handling for pre 1.1.25; skips missing fields in the schema, in order + if skip_missing_fields and label in missing_keys: + continue # skip incrementing the src_index so it stays in place, in order + if oskeys[src_index] != label: + raise MissingFieldError(f"Missing or out-of-order field = {label} " + f"from = {list(osalls)} in sad.") + src_index += 1 # advance source index only if label found in source oskeys # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion saids = copy.copy(fields.saids) # get copy of saidive field labels and defaults values - if not (set(saids.keys()) <= set(alls.keys())): + if not (set(saids) <= set(alls)): raise MissingFieldError(f"Missing one or more required said fields" - f" from {list(saids.keys())} in sad = " + f" from {list(saids)} in sad = " f"{self._sad}.") sad = self.sad # make shallow copy so don't clobber original .sad - for label in saids.keys(): + for label in saids: try: # replace default code with code of value from sad saids[label] = Matter(qb64=sad[label]).code except Exception as ex: @@ -557,33 +772,48 @@ def _verify(self): if saids[label] in DigDex: # if digestive then replace with dummy sad[label] = self.Dummy * len(sad[label]) - + # compute saidive digestive field values using raw from sized dummied sad raw = self.dumps(sad, kind=self.kind) # serialize dummied sad copy - for label, code in saids.items(): if code in DigDex: # subclass override if non digestive allowed - klas, size, length = self.Digests[code] # digest algo size & length - ikwa = dict() # digest algo class initi keyword args - if size: - ikwa.update(digest_size=size) # optional digest_size - dkwa = dict() # digest method keyword args - if length: - dkwa.update(length=length) - dig = Matter(raw=klas(raw, **ikwa).digest(**dkwa), code=code).qb64 + dig = Diger(ser=raw, code=code).qb64 if dig != self._sad[label]: # compare to original raise ValidationError(f"Invalid said field '{label}' in sad" f" = {self._sad}, should be {dig}.") sad[label] = dig - raw = self.dumps(sad, kind=self.kind) + raw = self.dumps(sad, kind=self.kind) # compute final raw + if raw != self.raw: raise ValidationError(f"Invalid round trip of {sad} != \n" f"{self.sad}.") + + if "v" not in sad: + raise ValidationError(f"Missing version string field in {sad}.") + + # extract version string elements to verify consistency with attributes + proto, vrsn, kind, size, opt = deversify(sad["v"]) + if self.proto != proto: + raise ValidationError(f"Inconsistent protocol={self.proto} in {sad}.") + + if self.vrsn != vrsn: + raise ValidationError(f"Inconsistent version={self.vrsn} in {sad}.") + + if self.kind != kind: + raise ValidationError(f"Inconsistent kind={self.kind} in {sad}.") + + if self.kind in (Kinds.json, Kinds.cbor, Kinds.mgpk): + if size != self.size != len(raw): + raise ValidationError(f"Inconsistent size={self.size} in {sad}.") + else: # size is not set in version string when kind is CESR + if self.size != len(raw): + raise ValidationError(f"Inconsistent size={self.size} in {sad}.") + # verified successfully since no exception - def makify(self, sad, *, version=None, - proto=None, vrsn=None, kind=None, ilk=None, saids=None): + def makify(self, sad, *, proto=None, vrsn=None, kind=None, + ilk=None, saids=None): """Makify given sad dict makes the versions string and computes the said field values and sets associated properties: raw, sad, proto, version, kind, size @@ -601,9 +831,7 @@ def makify(self, sad, *, version=None, Parameters: sad (dict): serializable saidified field map of message. Ignored if raw provided - version (Versionage): instance supported protocol version - None means do not enforce version - proto (str | None): desired protocol type str value of Protos + proto (str | None): desired protocol type str value of Protocols If None then its extracted from sad or uses default .Proto vrsn (Versionage | None): instance desired protocol version If None then its extracted from sad or uses default .Vrsn @@ -623,80 +851,115 @@ def makify(self, sad, *, version=None, sproto = svrsn = skind = silk = None if sad and 'v' in sad: # attempt to get from vs in sad try: # extract version string elements as defaults if provided - sproto, svrsn, skind, _ = deversify(sad["v"], version=version) - except ValueError as ex: + sproto, svrsn, skind, _, _ = deversify(sad["v"]) + except VersionError as ex: pass else: - silk = sad.get('t') # if not in get returns None which may be valid + silk = sad.get('t') # if 't' not in sad .get returns None which may be valid if proto is None: proto = sproto if sproto is not None else self.Proto - if vrsn is None: - vrsn = svrsn if svrsn is not None else self.Vrsn + if proto not in self.Fields: + raise SerializeError(f"Invalid protocol={proto}.") - if kind is None: - kind = skind if skind is not None else self.Kind + if self.Protocol and proto != self.Protocol: # required by class + raise SerializeError(f"Required protocol={self.Protocol}, got " + f"protocol={proto} instead.") - if ilk is None: - ilk = silk if silk is not None else self.Ilks[proto] + if self.genus not in GenDex: # ensures self.genus != None + raise SerializeError(f"Invalid genus={self.genus}.") + if getattr(GenDex, proto, None) != self.genus: + raise SerializeError(f"Incompatible protocol={proto} with " + f"genus={self.genus}.") - if proto not in self.Fields: - raise SerializeError(f"Invalid protocol type = {proto}.") + if vrsn is None: + vrsn = svrsn if svrsn is not None else self.Vrsn + if vrsn not in self.Fields[proto]: + raise SerializeError(f"Invalid version={vrsn} for protocol={proto}.") - if self.Protocol and proto != self.Protocol: - raise SerializeError(f"Expected protocol = {self.Protocol}, got " - f"{proto} instead.") + if vrsn.major > self.gvrsn.major: + raise SerializeError(f"Incompatible major protocol version={vrsn} " + f"with major genus version={self.gvrsn}.") - if version is not None and vrsn != version: - raise SerializeError(f"Expected version = {version}, got " - f"{vrsn.major}.{vrsn.minor}.") + if kind is None: + kind = skind if skind is not None else self.Kind - if kind not in Serials: - raise SerializeError(f"Invalid serialization kind = {kind}") + if ilk is None: # default is first ilk in Fields for given proto vrsn + ilk = (silk if silk is not None else + list(self.Fields[proto][vrsn])[0]) # list(dict) gives list of keys + if kind not in Kinds: + raise SerializeError(f"Invalid serialization kind = {kind}") if ilk not in self.Fields[proto][vrsn]: raise SerializeError(f"Invalid packet type (ilk) = {ilk} for" f"protocol = {proto}.") - fields = self.Fields[proto][vrsn][ilk] # get Fieldage of fields + fields = self.Fields[proto][vrsn][ilk] # get FieldDom of fields + + alls = fields.alls # faster local reference + oalls = oset(alls) # ordereset of field labels + oopts = oset(fields.opts) # ordereset of field labels + oreqs = oalls - oopts # required fields + - if not sad: # empty or None so create from defaults + if not sad: # empty or None so create sad dict sad = {} - for label, value in fields.alls.items(): - if helping.nonStringIterable(value): # copy iterable defaults - value = copy.copy(value) + + # ensure all required fields are in sad. If not provide default + for label in oreqs: + if label not in sad: + value = alls[label] + if helping.nonStringIterable(value): + value = copy.copy(value) # copy iterable defaults sad[label] = value - if 't' in sad: # packet type (ilk) requried so set value to ilk - sad['t'] = ilk + sadold = sad + sad = {} + for label in oalls: # make sure all fields are in correct order + if label in sadold: + sad[label] = sadold[label] - # ensure all required fields in alls are in sad - alls = fields.alls # all field labels - for label, value in alls.items(): # ensure provided sad as all required fields - if label not in sad: # supply default - if helping.nonStringIterable(value): # copy iterable defaults - value = copy.copy(value) - sad[label] = value + for label in sadold: # copy extras if any + if label not in sad: + sad[label] = sadold[label] + + if 't' in sad: # when packet type field then force ilk + sad['t'] = ilk # assign ilk + + # ensure required fields are present and all fields are ordered wrt alls + oskeys = oset(sad) # ordered set of keys in sad (skeys) + osexts = oskeys - oalls # get ordered set of extras in sad (sexts) + if osexts and fields.strict: + raise SerializeError(f"Unallowed extra field(s) = {list(osexts)} " + f"in sad.") - keys = list(sad) # get list of keys of self.sad - for key in list(keys): # make copy to mutate - if key not in alls: - del keys[keys.index(key)] # remove non required fields + osopts = oskeys - oreqs - osexts # subset of opts in sad - if list(alls.keys()) != keys: # ensure ordering of fields matches alls - raise SerializeError(f"Mismatch one or more of all required fields " - f" = {list(alls.keys())} in sad = {sad}.") + osalls = oalls - (oopts - osopts) # subset of alls without missing opts in sad + + for k, v in fields.alts.items(): + if k in osopts and v in osopts: + raise SerializeError(f"Unallowed, alternate fields '{k}' " + f"and '{v}' both present in sad.") + + # can't do set math osalls == oskeys - osexts becasue of osexts might be + # out-of-order so have to iterate to ensure osexts if any appear in oskeys + # after all fields in osalls + for i, label in enumerate(osalls): + if oskeys[i] != label: + raise SerializeError(f"Missing or out-of-order field = {label} " + f"from = {list(osalls)} in sad.") # said field labels are not order dependent with respect to all fields # in sad so use set() to test inclusion _saids = copy.copy(fields.saids) # get copy of defaults - if not (set(_saids.keys()) <= set(alls.keys())): + if not (set(_saids) <= set(alls)): raise SerializeError(f"Missing one or more required said fields " - f"from {list(_saids.keys())} in sad = {sad}.") + f"from {list(_saids)} in sad = {sad}.") # override saidive defaults for label in _saids: @@ -716,32 +979,29 @@ def makify(self, sad, *, version=None, raise SerializeError(f"Missing requires version string field 'v'" f" in sad = {sad}.") - sad['v'] = self.Dummy * VERFULLSIZE # ensure size of vs + if kind in (Kinds.json, Kinds.cbor, Kinds.mgpk): + # this size of sad needs to be computed based on actual version string span + # since not same for all versions + sad['v'] = self.Dummy * self.Spans[vrsn] # ensure span of vs is dummied MAXVERFULLSPAN - raw = self.dumps(sad, kind) # get size of fully dummied sad - size = len(raw) + raw = self.dumps(sad, kind) # get size of sad with fully dummied vs and saids + size = len(raw) - # generate new version string with correct size - vs = versify(proto=proto, version=vrsn, kind=kind, size=size) - sad["v"] = vs # update version string in sad + # generate new version string with correct size + vs = versify(protocol=proto, version=vrsn, kind=kind, size=size) + sad["v"] = vs # update version string in sad + # now have correctly sized version string in sad - # now have correctly sized version string in sad - # now compute saidive digestive field values using sized dummied sad - raw = self.dumps(sad, kind=kind) # serialize sized dummied sad + # compute saidive digestive field values using raw from sized dummied sad + raw = self.dumps(sad, kind=kind, proto=proto, vrsn=vrsn) # serialize sized dummied sad for label, code in _saids.items(): if code in DigDex: # subclass override if non digestive allowed - klas, dsize, dlen = self.Digests[code] # digest algo size & length - ikwa = dict() # digest algo class initi keyword args - if dsize: - ikwa.update(digest_size=dsize) # optional digest_size - dkwa = dict() # digest method keyword args - if dlen: - dkwa.update(length=dlen) - dig = Matter(raw=klas(raw, **ikwa).digest(**dkwa), code=code).qb64 - sad[label] = dig + sad[label] = Diger(ser=raw, code=code).qb64 - raw = self.dumps(sad, kind=kind) # compute final raw + raw = self.dumps(sad, kind=kind, proto=proto, vrsn=vrsn) # compute final raw + if kind == Kinds.cesr:# cesr kind version string does not set size + size = len(raw) # size of whole message self._raw = raw self._sad = sad @@ -749,19 +1009,9 @@ def makify(self, sad, *, version=None, self._vrsn = vrsn self._kind = kind self._size = size - # primary said field label - try: - label = list(self.Fields[self.proto][self.vrsn][self.ilk].saids.keys())[0] - if label not in self._sad: - raise SerializeError(f"Missing primary said field in {self._sad}.") - self._said = self._sad[label] # implicitly verified - except Exception: - self._said = None # no saidive field - - @classmethod - def _inhale(clas, raw, version=Version, reaped=None): + def _inhale(self, raw, *, smellage=None): """Deserializes raw. Parses serilized event ser of serialization kind and assigns to instance attributes and returns tuple of associated elements. @@ -772,96 +1022,77 @@ def _inhale(clas, raw, version=Version, reaped=None): Returns: tuple (sad, proto, vrsn, kind, size) where: sad (dict): serializable attribute dict of saidified data - proto (str): value of Protos (Protocolage) protocol type + proto (str): value of Protocols (Protocolage) protocol type vrsn (Versionage | None): tuple of (major, minor) version ints None means do not enforce version kind (str): value of Serials (Serialage) serialization kind Parameters: + clas (Serder): class reference raw (bytes): serialized sad message - version (Versionage): instance supported protocol version - reaped (Reapage | None): instance of deconstructed version string + smellage (Smellage | None): instance of deconstructed version string elements. If none or empty ignore otherwise assume that raw already had its version string extracted (reaped) into the - elements of reaped. - - Note: - loads and jumps of json use str whereas cbor and msgpack use bytes - Assumes only supports Version - - """ - if reaped: - proto, major, minor, kind, size = reaped # tuple unpack - else: - if len(raw) < clas.InhaleSize: - raise ShortageError(f"Need more raw bytes for Serder to inhale.") - - match = Rever.search(raw) # Rever regex takes bytes/bytearray not str - if not match or match.start() > clas.MaxVSOffset: - raise VersionError(f"Invalid version string in raw = " - f"{raw[:clas.InhaleSize]}.") - - proto, major, minor, kind, size = match.group("proto", - "major", - "minor", - "kind", - "size") + elements of smellage. - proto = proto.decode("utf-8") - if proto not in Protos: - raise ProtocolError(f"Invalid protocol type = {proto}.") - vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) - if version is not None and vrsn != version: - raise VersionError(f"Expected version = {version}, got " - f"{vrsn.major}.{vrsn.minor}.") - kind = kind.decode("utf-8") - if kind not in Serials: - raise KindError(f"Invalid serialization kind = {kind}.") - - size = int(size, 16) - if len(raw) < size: - raise ShortageError(f"Need more bytes.") + """ + if smellage: # passed in so don't need to smell raw again + proto, vrsn, kind, size, gvrsn = smellage # tuple unpack + else: # not passed in so smell raw + proto, vrsn, kind, size, gvrsn = smell(raw) - sad = clas.loads(raw=raw, size=size, kind=kind) + sad = self.loads(raw=raw, size=size, kind=kind) + # ._gvrsn may be set in loads when CESR native deserialization provides _gvrsn - if "v" not in sad: + if "v" not in sad: # Regex does not check for version string label itself raise FieldError(f"Missing version string field in {sad}.") - return sad, proto, vrsn, kind, size + # cypto opts want bytes not bytearray + self._raw = bytes(raw[:size]) # make copy so strip not affect + self._sad = sad + self._proto = proto + self._vrsn = vrsn + self._kind = kind + self._size = size - @staticmethod - def loads(raw, size=None, kind=Serials.json): - """Utility static method to handle deserialization by kind + + def loads(self, raw, size=None, kind=Kinds.json): + """method to handle deserialization by kind + assumes already sniffed and smelled to determine + serialization size and kind Returns: sad (dict | list): deserialized dict or list. Assumes attribute dict of saidified data. Parameters: - raw (bytes |bytearray): raw serialization to deserialze as dict + raw (bytes | bytearray): raw serialization to deserialze as dict size (int): number of bytes to consume for the deserialization. If None then consume all bytes in raw kind (str): value of Serials (Serialage) serialization kind "JSON", "MGPK", "CBOR" + + Notes: + loads of json uses str whereas loads of cbor and msgpack use bytes """ - if kind == Serials.json: + if kind == Kinds.json: try: sad = json.loads(raw[:size].decode("utf-8")) except Exception as ex: raise DeserializeError(f"Error deserializing JSON: " f"{raw[:size].decode('utf-8')}") from ex - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: try: sad = msgpack.loads(raw[:size]) except Exception as ex: raise DeserializeError(f"Error deserializing MGPK: " f"{raw[:size].decode('utf-8')}") from ex - elif kind == Serials.cbor: + elif kind == Kinds.cbor: try: sad = cbor.loads(raw[:size]) except Exception as ex: @@ -874,83 +1105,275 @@ def loads(raw, size=None, kind=Serials.json): return sad - @classmethod - def _exhale(clas, sad, version=None): - """Serializes sad given kind and version and sets the serialized size - in the version string. - - As classmethod enables bootstrap of valid sad dict that has correct size - in version string. This obviates sizeify. This can be called on self as - well because it only ever accesses clas attributes not instance attributes. + def _loads(self, raw, size=None): + """CESR native desserialization of raw - Returns tuple of (raw, proto, kind, sad, vrsn) where: - raw (str): serialized event as bytes of kind - proto (str): protocol type as value of Protocolage - kind (str): serialzation kind as value of Serialage - sad (dict): modified serializable attribute dict of saidified data - vrsn (Versionage): tuple value (major, minor) + Returns: + sad (dict): deserialized dict of CESR native serialization. Parameters: - sad (dict): serializable attribute dict of saidified data - version (Versionage | None): supported protocol version for message - None means do not enforce a supported version + clas (Serder): class reference + raw (bytes |bytearray): raw serialization to deserialze as dict + size (int): number of bytes to consume for the deserialization. + If None then consume all bytes in raw + """ + # ._gvrsn may be set in loads when CESR native deserialization provides _gvrsn + pass + + def _exhale(self, sad): + """Serializes sad and assigns attributes. + Asssumes all field values in sad are valid. + Call .verify to otherwise + Parameters: + sad (dict): serializable attribute dict of saidified data """ if "v" not in sad: raise SerializeError(f"Missing version string field in {sad}.") # extract elements so can replace size element but keep others - proto, vrsn, kind, size = deversify(sad["v"], version=version) - - raw = clas.dumps(sad, kind) - size = len(raw) - - # generate new version string with correct size - vs = versify(proto=proto, version=vrsn, kind=kind, size=size) + proto, vrsn, kind, size, opt = deversify(sad["v"]) - # find location of old version string inside raw - match = Rever.search(raw) # Rever's regex takes bytes - if not match or match.start() > 12: - raise SerializeError(f"Invalid version string in raw = {raw}.") - fore, back = match.span() # start and end positions of version string + raw = self.dumps(sad, kind) - # replace old version string in raw with new one - raw = b'%b%b%b' % (raw[:fore], vs.encode("utf-8"), raw[back:]) - if size != len(raw): # substitution messed up - raise SerializeError(f"Malformed size of raw in version string == {vs}") - sad["v"] = vs # update sad + if kind in (Kinds.cesr): # cesr kind version string does not set size + size = len(raw) # size of whole message - return raw, sad, proto, vrsn, kind, size + # must call .verify to ensure these are compatible + self._raw = raw # crypto opts want bytes not bytearray + self._sad = sad + self._proto = proto + self._vrsn = vrsn + self._kind = kind + self._size = size - @staticmethod - def dumps(sad, kind=Serials.json): - """Utility static method to handle serialization by kind + def dumps(self, sad=None, kind=Kinds.json, proto=None, vrsn=None): + """Method to handle serialization by kind + Assumes sad fields are properly filled out for serialization kind. Returns: - raw (bytes): serialization of sad dict using serialization kind + raw (bytes): serialization of sad dict using serialization kind Parameters: - sad (dict | list)): serializable dict or list to serialize - kind (str): value of Serials (Serialage) serialization kind - "JSON", "MGPK", "CBOR" + sad (dict | list | None)): serializable dict or list to serialize + kind (str): value of Serials (Serialage) serialization kind + "JSON", "MGPK", "CBOR", "CSER" + proto (str | None): desired protocol type str value of Protocols + If None then eventually use self.proto + vrsn (Versionage | None): instance desired protocol version + If None then eventually self.vrsn + + + Notes: + dumps of json uses str whereas dumps of cbor and msgpack use bytes + crypto opts want bytes not bytearray """ - if kind == Serials.json: + sad = sad if sad is not None else self.sad + + if kind == Kinds.json: raw = json.dumps(sad, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - elif kind == Serials.mgpk: + elif kind == Kinds.mgpk: raw = msgpack.dumps(sad) - elif kind == Serials.cbor: + elif kind == Kinds.cbor: raw = cbor.dumps(sad) + + elif kind == Kinds.cesr: # does not support list only dict + raw = self._dumps(sad, proto=proto, vrsn=vrsn) + else: raise SerializeError(f"Invalid serialization kind = {kind}") return raw + def _dumps(self, sad=None, proto=None, vrsn=None): + """CESR native serialization of sad + + Returns: + raw (bytes): CESR native serialization of sad dict + + Parameters: + sad (dict | None)): serializable dict to serialize + proto (str | None): desired protocol type str value of Protocols + If None then self.proto + vrsn (Versionage | None): instance desired protocol version + If None then self.vrsn + + Versioning: + CESR native serialization includes in its fixed version field + a version primitive that includes message protocol+protocol version + +genus version: 0NPPPPMmmMmm (12 B64 characters) + + This assumes that genus is compatible with message + protocol so genus is not needed. This protects from malleability attack + and ensure compatible cesr codes especially count (group) codes. + Primitive codes are less problematic since so far all primitive codes + tables are backwards compatible across major versions. + + """ + sad = sad if sad is not None else self.sad + proto = proto if proto is not None else self.proto + vrsn = vrsn if vrsn is not None else self.vrsn + + if (self.gvrsn.major < Vrsn_2_0.major or vrsn.major < Vrsn_2_0.major): + raise SerializeError(f"Invalid major genus version={self.gvrsn}" + f"or Invalid major protocol version={vrsn}" + f" for native CESR serialization.") + + if self.genus not in GenDex: # ensures self.genus != None + raise SerializeError(f"Invalid genus={self.genus}.") + + if getattr(GenDex, proto, None) != self.genus: + raise SerializeError(f"Incompatible protocol={proto} with " + f"genus={self.genus}.") + + + + + raw = bytearray() # message as qb64 + bdy = bytearray() # message body as qb64 + ilks = self.Fields[proto][vrsn] # get fields keyed by ilk + + ilk = sad.get('t') # returns None if missing message type (ilk) + if ilk not in ilks: # + raise SerializeError(f"Missing message type field " + f"'t' for protocol={proto} " + f"version={vrsn} with {sad=}.") + + fields = ilks[ilk] # FieldDom for given protocol and ilk + + if fields.opts or not fields.strict: # optional or extra fields allowed + fixed = False # so must use field map + else: + fixed = True #fixed field + + + # assumes that sad's field ordering and field inclusion is correct + # so can serialize in order to compute saidive fields + # need to fix ._verify and .makify to account for CESR native serialization + + if proto == Protocols.keri: + if not fixed: # prepend label + pass # raise error + + for l, v in sad.items(): # assumes valid field order & presence + match l: # label + case "v": # protocol+version do not use version string itself + val = Verser(proto=proto, vrsn=vrsn).qb64b + + case "t": # message type (ilk), already got ilk + val = Ilker(ilk=v).qb64b # assumes same + + case "d" | "i" | "p" | "di": # said or aid + val = v.encode("utf-8") # already primitive qb64 make qb6b + + case "s" | "bt": # sequence number or numeric threshold + val = coring.Number(numh=v).qb64b # convert hex str + + case "kt" | "nt": # current or next signing threshold + val = coring.Tholder(sith=v).limen # convert sith str + + case "k" | "n" | "b" | "ba" | "br": # list of primitives + frame = bytearray() + for e in v: # list + frame.extend(e.encode("utf-8")) + + val = bytearray(Counter(Codens.GenericListGroup, + count=len(frame) // 4).qb64b) + val.extend(frame) + + case "c": # list of config traits strings + frame = bytearray() + for e in v: # list + frame.extend(Traitor(trait=e).qb64b) + + val = bytearray(Counter(Codens.GenericListGroup, + count=len(frame) // 4).qb64b) + val.extend(frame) + + case "a": # list of seals or field map of attributes + frame = bytearray() # whole list + gcode = None # code for counter for consecutive same type seals + gframe = bytearray() # consecutive same type seals + for e in v: # list of seal dicts + # need support for grouping consecutive seals of same type with same counter + + try: + sealer = Sealer(crew=e) + code = self.ClanCodes[sealer.name] + if gcode and gcode == code: + gframe.extend(sealer.qb64b) + else: + if gframe: # not same so close off and rotate group + counter = Counter(code=gcode, count=len(gframe) // 4) + frame.extend(counter.qb64b + gframe) + gframe = bytearray() # new group + gcode = code # new group or keep same group + gframe.extend(sealer.qb64b) # extend in new group + + except kering.InvalidValueError: + if gframe: + counter = Counter(code=gcode, count=len(gframe) // 4) + frame.extend(counter.qb64b + gframe) + gframe = bytearray() + gcode = None + + #unknown seal type so serialize as field map + #generic seal no count type (v, Mapping): + #for l, e in v.items(): + #pass + #val = bytearray(Counter(tag=""GenericMapGroup"", + # count=len(frame) // 4).qb64b) + #val.extend(mapframe) + + if gframe: # close off last group if any + counter = Counter(code=gcode, count=len(gframe) // 4) + frame.extend(counter.qb64b + gframe) + gframe = bytearray() + gcode = None + + val = bytearray(Counter(Codens.GenericListGroup, + count=len(frame) // 4).qb64b) + val.extend(frame) + + case _: # if extra fields this is where logic would be + raise SerializeError(f"Unsupported protocol field label" + f"='{l}' for protocol={proto}" + f" version={vrsn}.") + + bdy.extend(val) + + + elif proto == Protocols.acdc: + for l, val in sad.items(): # assumes valid field order & presence + if not fixed: + pass # prepend label + + + + else: + raise SerializeError(f"Unsupported protocol={self.proto}.") + + + # prepend count code for message + if fixed: + + raw = bytearray(Counter(Codens.FixedMessageBodyGroup, + count=len(bdy) // 4).qb64b) + raw.extend(bdy) + else: + pass + + + return raw + + + def compare(self, said=None): """Utility method to allow comparison of own .said digest of .raw with some other purported said of .raw @@ -1012,6 +1435,26 @@ def sad(self): return dict(self._sad) # return copy + @property + def genus(self): + """genus (CESR genu code) property getter + + Returns: + genus (stre): CESR genus code for this Serder + """ + return self._genus + + + @property + def gvrsn(self): + """gvrsn (CESR genus version) property getter + + Returns: + gvrsn (Versionage): instance, CESR genus code table version for this Serder + """ + return self._gvrsn + + @property def kind(self): """kind property getter @@ -1022,7 +1465,7 @@ def kind(self): @property def proto(self): - """proto property getter + """proto property getter, protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' Returns: @@ -1030,6 +1473,16 @@ def proto(self): """ return self._proto + @property + def protocol(self): + """protocp; property getter, alias of .proto + protocol identifier type value of Protocolage such as 'KERI' or 'ACDC' + + Returns: + protocol (str): Protocolage value as protocol type + """ + return self.proto + @property def vrsn(self): @@ -1042,7 +1495,7 @@ def vrsn(self): @property def version(self): - """version property getter alias of .vrsn + """version property getter, alias of .vrsn Returns: version (Versionage): instance of protocol version for this Serder @@ -1065,7 +1518,7 @@ def said(self): Returns: said (str): qb64 """ - if not self.Fields[self.proto][self.vrsn][self.ilk].saids.keys() and 'd' in self._sad: + if not self.Fields[self.proto][self.vrsn][self.ilk].saids and 'd' in self._sad: return self._sad['d'] # special case for non-saidive messages like rct return self._said @@ -1096,9 +1549,8 @@ class SerderKERI(Serder): See docs for Serder """ #override in subclass to enforce specific protocol - Protocol = Protos.keri # required protocol, None means any in Protos is ok - Proto = Protos.keri # default protocol type - + Protocol = Protocols.keri # required protocol, None means any in Protocols is ok + Proto = Protocols.keri # default protocol type def _verify(self, **kwa): @@ -1112,17 +1564,16 @@ def _verify(self, **kwa): """ super(SerderKERI, self)._verify(**kwa) - allkeys = list(self.Fields[self.proto][self.vrsn][self.ilk].alls.keys()) - keys = list(self.sad.keys()) + allkeys = list(self.Fields[self.proto][self.vrsn][self.ilk].alls) + keys = list(self.sad) if allkeys != keys: - # special handling for 1.1.18 -> 1.1.32 + # special handling for pre 1.1.25 KERIpy not sending 'rp' field in exn messages. skip_missing_field = False if self.ilk == Ilks.exn: missing_keys = [key for key in allkeys if key not in keys] if missing_keys: if len(missing_keys) == 1 and 'rp' in missing_keys: skip_missing_field = True - if skip_missing_field: pass else: @@ -1131,8 +1582,8 @@ def _verify(self, **kwa): if (self.vrsn.major < 2 and self.vrsn.minor < 1 and self.ilk in (Ilks.qry, Ilks.rpy, Ilks.pro, Ilks.bar, Ilks.exn)): - pass - else: # verify pre + pass # non prefixive ilks do not have 'i' field + else: # verify pre 'i' field try: code = Matter(qb64=self.pre).code except Exception as ex: @@ -1147,19 +1598,38 @@ def _verify(self, **kwa): if code not in idex: raise ValidationError(f"Invalid identifier prefix code = {code}.") - # non-transferable pre validations - if code in [PreDex.Ed25519N, PreDex.ECDSA_256r1N, PreDex.ECDSA_256k1N]: - if self.ndigs: - raise ValidationError(f"Non-transferable code = {code} with" - f" non-empty nxt = {self.ndigs}.") - if self.backs: - raise ValidationError("Non-transferable code = {code} with" - f" non-empty backers = {self.backs}.") + if self.ilk in (Ilks.icp, Ilks.dip, Ilks.rot, Ilks.drt): # est event + if self.ilk in (Ilks.icp, Ilks.dip): # inceptive event + if code in PreNonDigDex: + if len(self.keys) != 1: + raise ValidationError(f"Invalid keys = {self.keys} " + "for non-digestive prefix " + f"{code=}.") - if self.seals: - raise ValidationError("Non-transferable code = {code} with" - f" non-empty seals = {self.seals}.") + if self.tholder.sith != '1': + raise ValidationError(f"Invalid signing threshold =" + f" {self.tholder.sith} for " + f"non-digestive prefix {code=}.") + + if self.pre != self.keys[0]: + raise ValidationError(f"Mismatch prefix = {self.pre} and" + f" zeroth key = {self.keys[0]} for " + f" non-digestive prefix {code=}.") + + # non-transferable pre validations + if code in NonTransDex: + if self.ndigs: # when field missing returns None + raise ValidationError(f"Non-transferable code = {code} with" + f" non-empty nxt = {self.ndigs}.") + + if self.backs: # when field missing returns None + raise ValidationError("Non-transferable code = {code} with" + f" non-empty backers = {self.backs}.") + + if self.seals: # when field missing returns None + raise ValidationError("Non-transferable code = {code} with" + f" non-empty seals = {self.seals}.") if self.ilk in (Ilks.dip): # validate delpre try: @@ -1469,58 +1939,6 @@ def nonce(self): return self.uuid -class SerderCREL(Serder): - """SerderCREL is Serder subclass with Labels for CREL packet types (ilks) and - properties for exposing field values of CREL messages - Container Registry Event Log for issuance, revocation, etc registries of - ACDC - - See docs for Serder - """ - #override in subclass to enforce specific protocol - Protocol = Protos.crel # required protocol, None means any in Protos is ok - Proto = Protos.crel # default protocol type - Vrsn = Vrsn_1_1 # default protocol version for protocol type - - - def _verify(self, **kwa): - """Verifies said(s) in sad against raw - Override for protocol and ilk specific verification behavior. Especially - for inceptive ilks that have more than one said field like a said derived - identifier prefix. - - Raises a ValidationError (or subclass) if any verification fails - - """ - super(SerderCREL, self)._verify(**kwa) - - try: - code = Matter(qb64=self.issuer).code - except Exception as ex: - raise ValidationError(f"Invalid issuer AID = " - f"{self.issuer}.") from ex - - if code not in PreDex: - raise ValidationError(f"Invalid issuer AID code = {code}.") - - - @property - def issuer(self): - """ - Returns: - issuer (str): qb64 of .sad["i"] issuer AID property getter - """ - return self._sad.get('i') - - - @property - def issuerb(self): - """ - Returns: - issuerb (bytes): qb64b of .issuer property getter as bytes - """ - return self.issuer.encode("utf-8") if self.issuer is not None else None - class SerderACDC(Serder): """SerderACDC is Serder subclass with Labels for ACDC packet types (ilks) and @@ -1529,8 +1947,8 @@ class SerderACDC(Serder): See docs for Serder """ #override in subclass to enforce specific protocol - Protocol = Protos.acdc # required protocol, None means any in Protos is ok - Proto = Protos.acdc # default protocol type + Protocol = Protocols.acdc # required protocol, None means any in Protocols is ok + Proto = Protocols.acdc # default protocol type diff --git a/src/keri/core/signing.py b/src/keri/core/signing.py new file mode 100644 index 000000000..7a4faa63e --- /dev/null +++ b/src/keri/core/signing.py @@ -0,0 +1,1057 @@ +# -*- coding: utf-8 -*- +""" +keri.core.signing module + +Provides support Signer class +""" +from dataclasses import dataclass, astuple, asdict + +import pysodium + +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat +from cryptography.hazmat.primitives.asymmetric import ec, utils + +from ..kering import (EmptyMaterialError, InvalidCodeError, InvalidSizeError, + InvalidValueError) + +from ..help import helping + +from .coring import (Tiers, ) +from .coring import (SmallVrzDex, LargeVrzDex, Matter, MtrDex, Verfer, Cigar) +from .indexing import IdrDex, Indexer, Siger +from .streaming import Streamer + + +DSS_SIG_MODE = "fips-186-3" +ECDSA_256r1_SEEDBYTES = 32 +ECDSA_256k1_SEEDBYTES = 32 + + + + +class Signer(Matter): + """ + Signer is Matter subclass with method to create signature of serialization + using: + .raw as signing (private) key seed, + .code as cipher suite for signing + .verfer whose property .raw is public key for signing. + + If not provided .verfer is generated from private key seed using .code + as cipher suite for creating key-pair. + + + See Matter for inherited attributes and properties: + + Attributes: + + Properties: (inherited) + code (str): hard part of derivation code to indicate cypher suite + both (int): hard and soft parts of full text code + size (int): Number of triplets of bytes including lead bytes + (quadlets of chars) of variable sized material. Value of soft size, + ss, part of full text code. + Otherwise None. + rize (int): number of bytes of raw material not including + lead bytes + raw (bytes): private signing key crypto material only without code + qb64 (str): private signing key Base64 fully qualified with + derivation code + crypto mat + qb64b (bytes): private signing keyBase64 fully qualified with + derivation code + crypto mat + qb2 (bytes): private signing key binary with + derivation code + crypto material + transferable (bool): True means transferable derivation code False otherwise + digestive (bool): True means digest derivation code False otherwise + + Properties: + + .verfer is Verfer object instance of public key derived from private key + seed which is .raw + + Methods: + sign: create signature + + """ + + def __init__(self, raw=None, code=MtrDex.Ed25519_Seed, transferable=True, **kwa): + """ + Assign signing cipher suite function to ._sign + + Parameters: See Matter for inherted parameters + raw is bytes crypto material seed or private key + code is derivation code + transferable is Boolean True means make verifier code transferable + False make non-transferable + + """ + try: + super(Signer, self).__init__(raw=raw, code=code, **kwa) + except EmptyMaterialError as ex: + if code == MtrDex.Ed25519_Seed: + raw = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + super(Signer, self).__init__(raw=raw, code=code, **kwa) + elif code == MtrDex.ECDSA_256r1_Seed: + raw = pysodium.randombytes(ECDSA_256r1_SEEDBYTES) + super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) + elif code == MtrDex.ECDSA_256k1_Seed: + raw = pysodium.randombytes(ECDSA_256k1_SEEDBYTES) + super(Signer, self).__init__(raw=bytes(raw), code=code, **kwa) + + else: + raise ValueError("Unsupported signer code = {}.".format(code)) + + if self.code == MtrDex.Ed25519_Seed: + self._sign = self._ed25519 + verkey, sigkey = pysodium.crypto_sign_seed_keypair(self.raw) + verfer = Verfer(raw=verkey, + code=MtrDex.Ed25519 if transferable + else MtrDex.Ed25519N) + elif self.code == MtrDex.ECDSA_256r1_Seed: + self._sign = self._secp256r1 + d = int.from_bytes(self.raw, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256R1()) + verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) + verfer = Verfer(raw=verkey, + code=MtrDex.ECDSA_256r1 if transferable + else MtrDex.ECDSA_256r1N) + elif self.code == MtrDex.ECDSA_256k1_Seed: + self._sign = self._secp256k1 + d = int.from_bytes(self.raw, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256K1()) + verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) + verfer = Verfer(raw=verkey, + code=MtrDex.ECDSA_256k1 if transferable + else MtrDex.ECDSA_256k1N) + else: + raise ValueError("Unsupported signer code = {}.".format(self.code)) + + self._verfer = verfer + + @property + def verfer(self): + """ + Property verfer: + Returns Verfer instance + Assumes ._verfer is correctly assigned + """ + return self._verfer + + def sign(self, ser, index=None, only=False, ondex=None, **kwa): + """ + Returns either Cigar or Siger (indexed) instance of cryptographic + signature material on bytes serialization ser + + If index is None + return Cigar instance + Else + return Siger instance + + Parameters: + ser (bytes): serialization to be signed + index (int): main index of associated verifier key in event keys + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + + """ + return (self._sign(ser=ser, + seed=self.raw, + verfer=self.verfer, + index=index, + only=only, + ondex=ondex, + **kwa)) + + @staticmethod + def _ed25519(ser, seed, verfer, index, only=False, ondex=None, **kwa): + """ + Returns signature as either Cigar or Siger instance as appropriate for + Ed25519 digital signatures given index and ondex values + + The seed's code determins the crypto key-pair algorithm and signing suite + The signature type, Cigar or Siger, and when indexed the Siger code + may be completely determined by the seed and index values (index, ondex) + by assuming that the index values are intentional. + Without the seed code its more difficult for Siger to + determine when for the Indexer code value should be changed from the + than the provided value with respect to provided but incompatible index + values versus error conditions. + + Parameters: + ser (bytes): serialization to be signed + seed (bytes): raw binary seed (private key) + verfer (Verfer): instance. verfer.raw is public key + index (int |None): main index offset into list such as current signing + None means return non-indexed Cigar + Not None means return indexed Siger with Indexer code derived + from index, conly, and ondex values + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + """ + # compute raw signature sig using seed on serialization ser + sig = pysodium.crypto_sign_detached(ser, seed + verfer.raw) + + if index is None: # Must be Cigar i.e. non-indexed signature + return Cigar(raw=sig, code=MtrDex.Ed25519_Sig, verfer=verfer) + else: # Must be Siger i.e. indexed signature + # should add Indexer class method to get ms main index size for given code + if only: # only main index ondex not used + ondex = None + if index <= 63: # (64 ** ms - 1) where ms is main index size + code = IdrDex.Ed25519_Crt_Sig # use small current only + else: + code = IdrDex.Ed25519_Big_Crt_Sig # use big current only + else: # both + if ondex == None: + ondex = index # enable default to be same + if ondex == index and index <= 63: # both same and small + code = IdrDex.Ed25519_Sig # use small both same + else: # otherwise big or both not same so use big both + code = IdrDex.Ed25519_Big_Sig # use use big both + + return Siger(raw=sig, code=code, index=index, ondex=ondex, verfer=verfer) + + @staticmethod + def _secp256r1(ser, seed, verfer, index, only=False, ondex=None, **kwa): + """ + Returns signature as either Cigar or Siger instance as appropriate for + Ed25519 digital signatures given index and ondex values + + The seed's code determins the crypto key-pair algorithm and signing suite + The signature type, Cigar or Siger, and when indexed the Siger code + may be completely determined by the seed and index values (index, ondex) + by assuming that the index values are intentional. + Without the seed code its more difficult for Siger to + determine when for the Indexer code value should be changed from the + than the provided value with respect to provided but incompatible index + values versus error conditions. + + Parameters: + ser (bytes): serialization to be signed + seed (bytes): raw binary seed (private key) + verfer (Verfer): instance. verfer.raw is public key + index (int |None): main index offset into list such as current signing + None means return non-indexed Cigar + Not None means return indexed Siger with Indexer code derived + from index, conly, and ondex values + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + """ + # compute raw signature sig using seed on serialization ser + d = int.from_bytes(seed, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256R1()) + der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) + (r, s) = utils.decode_dss_signature(der) + sig = bytearray(r.to_bytes(32, "big")) + sig.extend(s.to_bytes(32, "big")) + + if index is None: # Must be Cigar i.e. non-indexed signature + return Cigar(raw=sig, code=MtrDex.ECDSA_256r1_Sig, verfer=verfer) + else: # Must be Siger i.e. indexed signature + # should add Indexer class method to get ms main index size for given code + if only: # only main index ondex not used + ondex = None + if index <= 63: # (64 ** ms - 1) where ms is main index size + code = IdrDex.ECDSA_256r1_Crt_Sig # use small current only + else: + code = IdrDex.ECDSA_256r1_Big_Crt_Sig # use big current only + else: # both + if ondex == None: + ondex = index # enable default to be same + if ondex == index and index <= 63: # both same and small + code = IdrDex.ECDSA_256r1_Sig # use small both same + else: # otherwise big or both not same so use big both + code = IdrDex.ECDSA_256r1_Big_Sig # use use big both + + return Siger(raw=sig, code=code, index=index, ondex=ondex, verfer=verfer,) + + @staticmethod + def _secp256k1(ser, seed, verfer, index, only=False, ondex=None, **kwa): + """ + Returns signature as either Cigar or Siger instance as appropriate for + secp256k1 digital signatures given index and ondex values + + The seed's code determins the crypto key-pair algorithm and signing suite + The signature type, Cigar or Siger, and when indexed the Siger code + may be completely determined by the seed and index values (index, ondex) + by assuming that the index values are intentional. + Without the seed code its more difficult for Siger to + determine when for the Indexer code value should be changed from the + than the provided value with respect to provided but incompatible index + values versus error conditions. + + Parameters: + ser (bytes): serialization to be signed + seed (bytes): raw binary seed (private key) + verfer (Verfer): instance. verfer.raw is public key + index (int |None): main index offset into list such as current signing + None means return non-indexed Cigar + Not None means return indexed Siger with Indexer code derived + from index, conly, and ondex values + only (bool): True means main index only list, ondex ignored + False means both index lists (default), ondex used + ondex (int | None): other index offset into list such as prior next + """ + # compute raw signature sig using seed on serialization ser + d = int.from_bytes(seed, byteorder="big") + sigkey = ec.derive_private_key(d, ec.SECP256K1()) + der = sigkey.sign(ser, ec.ECDSA(hashes.SHA256())) + (r, s) = utils.decode_dss_signature(der) + sig = bytearray(r.to_bytes(32, "big")) + sig.extend(s.to_bytes(32, "big")) + + if index is None: # Must be Cigar i.e. non-indexed signature + return Cigar(raw=sig, code=MtrDex.ECDSA_256k1_Sig, verfer=verfer) + else: # Must be Siger i.e. indexed signature + # should add Indexer class method to get ms main index size for given code + if only: # only main index ondex not used + ondex = None + if index <= 63: # (64 ** ms - 1) where ms is main index size + code = IdrDex.ECDSA_256k1_Crt_Sig # use small current only + else: + code = IdrDex.ECDSA_256k1_Big_Crt_Sig # use big current only + else: # both + if ondex == None: + ondex = index # enable default to be same + if ondex == index and index <= 63: # both same and small + code = IdrDex.ECDSA_256k1_Sig # use small both same + else: # otherwise big or both not same so use big both + code = IdrDex.ECDSA_256k1_Big_Sig # use use big both + + return Siger(raw=sig, code=code, index=index, ondex=ondex, verfer=verfer,) + + + +class Salter(Matter): + """ + Salter is Matter subclass to maintain random salt for secrets (private keys) + Its .raw is random salt, .code as cipher suite for salt + + To initialize with deterministic salt pass in 16 bytes for raw: + salter = Salter(raw=b'0123456789abcdef') + + To create a deterministic secret, seed, or private key from salt + call .signer: + signer = salter.signer(code=MtrDex.Ed25519_Seed, + transferable=True, + path="", + tier=None, + temp=False) + + To create a deterministic set of secrets or seeds or private keys from salt + call signers: + signers = salter.signers(count=1, + start=0, + path="", + code=MtrDex.Ed25519_Seed, + transferable=True, + tier=None, + temp=False) + + Attributes: + .level is str security level code. Provides default level + + Inherited Properties + .pad is int number of pad chars given raw + .code is str derivation code to indicate cypher suite + .raw is bytes crypto material only without code + .index is int count of attached crypto material by context (receipts) + .qb64 is str in Base64 fully qualified with derivation code + crypto mat + .qb64b is bytes in Base64 fully qualified with derivation code + crypto mat + .qb2 is bytes in binary with derivation code + crypto material + .transferable is Boolean, True when transferable derivation code False otherwise + + Properties: + + Methods: + + Hidden: + ._pad is method to compute .pad property + ._code is str value for .code property + ._raw is bytes value for .raw property + ._index is int value for .index property + ._infil is method to compute fully qualified Base64 from .raw and .code + ._exfil is method to extract .code and .raw from fully qualified Base64 + + """ + Tier = Tiers.low + + def __init__(self, raw=None, code=MtrDex.Salt_128, tier=None, **kwa): + """ + Initialize salter's raw and code + + Inherited Parameters: + raw is bytes of unqualified crypto material usable for crypto operations + qb64b is bytes of fully qualified crypto material + qb64 is str or bytes of fully qualified crypto material + qb2 is bytes of fully qualified crypto material + code is str of derivation code + index is int of count of attached receipts for CryCntDex codes + + Parameters: + + """ + try: + super(Salter, self).__init__(raw=raw, code=code, **kwa) + except EmptyMaterialError as ex: + if code == MtrDex.Salt_128: + raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + super(Salter, self).__init__(raw=raw, code=code, **kwa) + else: + raise ValueError("Unsupported salter code = {}.".format(code)) + + if self.code not in (MtrDex.Salt_128,): + raise ValueError("Unsupported salter code = {}.".format(self.code)) + + self.tier = tier if tier is not None else self.Tier + + def stretch(self, *, size=32, path="", tier=None, temp=False): + """ + Returns (bytes): raw binary seed (secret) derived from path and .raw + and stretched to size given by code using argon2d stretching algorithm. + + Parameters: + size (int): number of bytes in stretched seed + path (str): unique chars used in derivation of seed (secret) + tier (str): value from Tierage for security level of stretch + temp is Boolean, True means use quick method to stretch salt + for testing only, Otherwise use time set by tier to stretch + """ + tier = tier if tier is not None else self.tier + + if temp: + opslimit = 1 # pysodium.crypto_pwhash_OPSLIMIT_MIN + memlimit = 8192 # pysodium.crypto_pwhash_MEMLIMIT_MIN + else: + if tier == Tiers.low: + opslimit = 2 # pysodium.crypto_pwhash_OPSLIMIT_INTERACTIVE + memlimit = 67108864 # pysodium.crypto_pwhash_MEMLIMIT_INTERACTIVE + elif tier == Tiers.med: + opslimit = 3 # pysodium.crypto_pwhash_OPSLIMIT_MODERATE + memlimit = 268435456 # pysodium.crypto_pwhash_MEMLIMIT_MODERATE + elif tier == Tiers.high: + opslimit = 4 # pysodium.crypto_pwhash_OPSLIMIT_SENSITIVE + memlimit = 1073741824 # pysodium.crypto_pwhash_MEMLIMIT_SENSITIVE + else: + raise ValueError("Unsupported security tier = {}.".format(tier)) + + # stretch algorithm is argon2id + seed = pysodium.crypto_pwhash(outlen=size, + passwd=path, + salt=self.raw, + opslimit=opslimit, + memlimit=memlimit, + alg=pysodium.crypto_pwhash_ALG_ARGON2ID13) + return (seed) + + def signer(self, *, code=MtrDex.Ed25519_Seed, transferable=True, path="", + tier=None, temp=False): + """ + Returns Signer instance whose .raw secret is derived from path and + salter's .raw and stretched to size given by code. The signers public key + for its .verfer is derived from code and transferable. + + Parameters: + code is str code of secret crypto suite + transferable is Boolean, True means use transferace code for public key + path is str of unique chars used in derivation of secret seed for signer + tier is str Tierage security level + temp is Boolean, True means use quick method to stretch salt + for testing only, Otherwise use more time to stretch + """ + seed = self.stretch(size=Matter._rawSize(code), path=path, tier=tier, + temp=temp) + + return (Signer(raw=seed, code=code, transferable=transferable)) + + + def signers(self, count=1, start=0, path="", **kwa): + """ + Returns list of count number of Signer instances with unique derivation + path made from path prefix and suffix of start plus offset for each count + value from 0 to count - 1. + + See .signer for parameters used to create each signer. + + """ + return [self.signer(path=f"{path}{i + start:x}", **kwa) for i in range(count)] + + + + +# Codes for for ciphers of variable sized sniffable QB2 or QB64 plain text +@dataclass(frozen=True) +class CipherX25519VarStrmCodex: + """ + CipherX25519VarCodex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is Sniffable CESR Stream. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarStrmDex = CipherX25519VarStrmCodex() # Make instance + + +# Codes for for ciphers of variable sized QB64 plain text +@dataclass(frozen=True) +class CipherX25519VarQB64Codex: + """ + CipherX25519VarQB64Codex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is QB64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarQB64Dex = CipherX25519VarQB64Codex() # Make instance + +# Codes for for ciphers of fixed sized QB64 plain text +@dataclass(frozen=True) +class CipherX25519FixQB64Codex: + """ + CipherX25519FixQB64Codex is codex all fixed sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is B64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + + def __iter__(self): + return iter(astuple(self)) + +CiXFixQB64Dex = CipherX25519FixQB64Codex() # Make instance + +# Codes for for ciphers of all sizes fixed and variable of QB64 plain text +@dataclass(frozen=True) +class CipherX25519AllQB64Codex: + """ + CipherX25519AllQB64Codex is codex all both fixed and variable sized cipher bytes + derivation codes for sealed box encryped ciphertext. Plaintext is B64. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXAllQB64Dex = CipherX25519AllQB64Codex() # Make instance + + + +# Codes for for ciphers of variable sized QB2 plain text +@dataclass(frozen=True) +class CipherX25519QB2VarCodex: + """ + CipherX25519QB2VarCodex is codex all variable sized cipher bytes derivation codes + for sealed box encryped ciphertext. Plaintext is B2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarQB2Dex = CipherX25519QB2VarCodex() # Make instance + +# Codes for for ciphers of all varibale sizes and all types of plain text +@dataclass(frozen=True) +class CipherX25519AllVarCodex: + """ + CipherX25519AllVarCodex is codex all variable size codes of cipher bytes + for sealed box encryped ciphertext. Plaintext maybe sniffable CESR stream or qb64 or qb2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXVarDex = CipherX25519AllVarCodex() # Make instance + + +# Codes for for ciphers of all sizes and all types of plain text +@dataclass(frozen=True) +class CipherX25519AllCodex: + """ + CipherX25519AllCodex is codex all codes and types of cipher bytes + for sealed box encryped ciphertext. Plaintext maybe sniffable or qb64 or qb2. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + X25519_Cipher_L0: str = '4C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 0 + X25519_Cipher_L1: str = '5C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 1 + X25519_Cipher_L2: str = '6C' # X25519 sealed box cipher bytes of sniffable stream plaintext lead size 2 + X25519_Cipher_Big_L0: str = '7AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 0 + X25519_Cipher_Big_L1: str = '8AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 1 + X25519_Cipher_Big_L2: str = '9AAC' # X25519 sealed box cipher bytes of sniffable stream plaintext big lead size 2 + X25519_Cipher_Seed: str = 'P' # X25519 sealed box 124 char qb64 Cipher of 44 char qb64 Seed + X25519_Cipher_Salt: str = '1AAH' # X25519 sealed box 100 char qb64 Cipher of 24 char qb64 Salt + X25519_Cipher_QB64_L0: str = '4D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 0 + X25519_Cipher_QB64_L1: str = '5D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 1 + X25519_Cipher_QB64_L2: str = '6D' # X25519 sealed box cipher bytes of QB64 plaintext lead size 2 + X25519_Cipher_QB64_Big_L0: str = '7AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 0 + X25519_Cipher_QB64_Big_L1: str = '8AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 1 + X25519_Cipher_QB64_Big_L2: str = '9AAD' # X25519 sealed box cipher bytes of QB64 plaintext big lead size 2 + X25519_Cipher_QB2_L0: str = '4E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 0 + X25519_Cipher_QB2_L1: str = '5E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 1 + X25519_Cipher_QB2_L2: str = '6E' # X25519 sealed box cipher bytes of QB2 plaintext lead size 2 + X25519_Cipher_QB2_Big_L0: str = '7AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 0 + X25519_Cipher_QB2_Big_L1: str = '8AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 1 + X25519_Cipher_QB2_Big_L2: str = '9AAE' # X25519 sealed box cipher bytes of QB2 plaintext big lead size 2 + + def __iter__(self): + return iter(astuple(self)) + +CiXDex = CipherX25519AllCodex() # Make instance + + + + +class Cipher(Matter): + """ + Cipher is Matter subclass holding a cipher text of a secret that may be + either a secret seed (private key) or secret salt with appropriate CESR code + to indicate which kind (which indicates size). The cipher text is created + with assymetric encryption using an unrelated (public, private) + encryption/decryption key pair. The public key is used for encryption the + private key for decryption. The default is to use X25519 sealed box encryption. + + The Cipher instances .raw is the raw binary encrypted cipher text and its + .code indicates what type of plain text has been encrypted. The cipher suite used + for the encryption/decryption is implied by the context where the cipher is + used. + + See Matter for inherited attributes and properties + + """ + Codex = CiXDex + Codes = asdict(CiXDex) # map code name to code + + def __init__(self, raw=None, code=None, **kwa): + """ + Inherited Parameters: + (see Matter) + + Parmeters: + raw (bytes | str): cipher text (not plain text) + code (str): cipher suite + + """ + # default when raw is not None and code is None is to use fixed size + # code given by raw size. Otherwise provided code fixed or variable size + # is handled by Matter superclass. + if raw is not None and code is None: + if len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Salt): + code = MtrDex.X25519_Cipher_Salt + elif len(raw) == Matter._rawSize(MtrDex.X25519_Cipher_Seed): + code = MtrDex.X25519_Cipher_Seed + else: + raise InvalidSizeError(f"Unsupported fixed raw size" + f" {len(raw)} for {code=}.") + + if hasattr(raw, "encode"): + raw = raw.encode("utf-8") # ensure bytes not str + + super(Cipher, self).__init__(raw=raw, code=code, **kwa) + + if self.code not in CiXDex: + raise InvalidCodeError(f"Unsupported cipher code = {self.code}.") + + + def decrypt(self, prikey=None, seed=None, klas=None, transferable=False, + bare=False, **kwa): + """ + Returns plain text as klas instance (Matter, Indexer, Streamer). + When klas is None then klas default is based on .code. Maybe Salter, + Signer, or Streamer. Encrypted plain text is fully + qualified (qb64) via self so derivaton code of plain text preserved through + encryption/decryption round trip. + + The created Decrypter uses either decryption key given by prikey or + when prikey missing derives prikey from signing key derived from private + seed. + + Returns: + decrypted (Matter | Indexer | Streamer): instance of decrypted + cipher text of .raw which is encrypted qb64, qb2, or sniffable + stream depending on .code when bare is False. Otherwise returns + plaintext itself. + + Keyword Parameters: + (see Matter because created Decrypter is Matter subclass) + + Parameters: + prikey (str | bytes): qb64 or qb64b serialization of private + decryption key. Must be fully qualified with code. + seed (str | bytes): qb64 or qb64b serialization of private + signing key seed used to derive private decryption key. Must be + fully qualified with code. + klas (Matter | Indexer | Streamer): Class used to create instance from + decrypted serialization. + transferable (bool): Modifier of klas instance creation. + When klas init (such as Signer) supports transferabe parm; + True means verfer of returned signer is transferable. + False means non-transferable + bare (bool): False (default) means returns instance holding plaintext + True means returns plaintext itself + """ + decrypter = Decrypter(qb64b=prikey, seed=seed, **kwa) + return decrypter.decrypt(cipher=self, + klas=klas, + transferable=transferable, + bare=bare) + + +class Encrypter(Matter): + """ + Encrypter is Matter subclass with method to create a cipher text of a + fully qualified (qb64) private key/seed where private key/seed is the plain + text. Encrypter uses assymetric (public, private) key encryption of a + serialization (plain text). Using its .raw as the encrypting (public) key and + its .code to indicate the cipher suite for the encryption operation. + + For example .code == MtrDex.X25519 indicates that X25519 sealed box + encyrption is used. The encryption key may be derived from an Ed25519 + signing public key that associated with a nontransferable or basic derivation + self certifying identifier. This allows use of the self certifying identifier + to track or manage the encryption/decryption key pair. And could be used to + provide additional authentication operations for using the + encryption/decryption key pair. Support for this is provided at init time + with the verkey parameter which allows deriving the encryption public key from + the fully qualified verkey (signature verification key). + + See Matter for inherited attributes and properties: + + Methods: + encrypt: returns cipher text + + """ + + def __init__(self, raw=None, code=MtrDex.X25519, verkey=None, **kwa): + """ + Assign encrypting cipher suite function to ._encrypt + + Parameters: See Matter for inherted parameters such as qb64, qb64b + raw (bytes): public encryption key + qb64b (bytes): fully qualified public encryption key + qb64 (str): fully qualified public encryption key + code (str): derivation code for public encryption key + verkey (Union[bytes, str]): qb64b or qb64 of verkey used to derive raw + """ + if not raw and verkey: + verfer = Verfer(qb64b=verkey) + if verfer.code not in (MtrDex.Ed25519N, MtrDex.Ed25519): + raise InvalidValueError(f"Unsupported verkey derivation code =" + f" {verfer.code}.") + # convert signing public key to encryption public key + raw = pysodium.crypto_sign_pk_to_box_pk(verfer.raw) + + super(Encrypter, self).__init__(raw=raw, code=code, **kwa) + + if self.code == MtrDex.X25519: + self._encrypt = self._x25519 + else: + raise InvalidValueError(f"Unsupported encrypter code = {self.code}.") + + def verifySeed(self, seed): + """ + Returns: + Boolean: True means private signing key seed corresponds to public + signing key verkey used to derive encrypter's .raw public + encryption key. + + Parameters: + seed (Union(bytes,str)): qb64b or qb64 serialization of private + signing key seed + """ + signer = Signer(qb64b=seed) + verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + return (pubkey == self.raw) + + def encrypt(self, *, ser=None, prim=None, code=None): + """ + Returns: + Cipher instance of cipher text encryption of plain text serialization + provided by either ser or prim as CESR primitive instance. + + Parameters: + + ser (str | bytes | bytearray | memoryview): qb64b or qb64 or sniffable + stream serialization of plain text + prim (Matter | Indexer | Streamer): CESR primitive instance whose + serialization is qb64 or qb2 or sniffable stream and is to be + encrypted based on code + code (str): code of plain text type for resultant encrypted cipher + """ + if not ser: + + if not prim: + raise EmptyMaterialError(f"Neither bar serialization or primitive " + f"are provided.") + + if not code: + if prim.code == MtrDex.Salt_128: # future other salt codes + code = MtrDex.X25519_Cipher_Salt + elif prim.code == MtrDex.Ed25519_Seed: # future other seed codes + code = MtrDex.X25519_Cipher_Seed + else: + raise InvalidValueError(f"Unsupported primitive with code =" + f" {prim.code} when cipher code is " + f"missing.") + + if code in CiXAllQB64Dex: + ser = prim.qb64b + elif code in CiXVarQB2Dex: + ser = prim.qb2 + elif code in CiXVarStrmDex: + ser = prim.stream + else: + raise InvalidCodeError(f"Invalid primitive cipher {code=} not " + f"qb64 or qb2.") + + if not code: # assumes default is sniffable stream + code = CiXDex.X25519_Cipher_L0 + + if hasattr(ser, "encode"): + ser = ser.encode() # convert str to bytes + if not isinstance(ser, bytes): + ser = bytes(ser) # convert bytearray and memoryview to bytes + + # encrypting cesr primitive qb64 or qb2 or cesr stream as plain + # text with proper cipher code ensures primitive round trip through eventual + # decryption. + return (self._encrypt(ser=ser, pubkey=self.raw, code=code)) + + @staticmethod + def _x25519(ser, pubkey, code): + """ + Returns cipher text as Cipher instance + Parameters: + ser (Union[bytes, str]): qb64b or qb64 serialization of seed or salt + to be encrypted. + pubkey (bytes): raw binary serialization of encryption public key + code (str): cipher derivation code + """ + raw = pysodium.crypto_box_seal(ser, pubkey) + return Cipher(raw=raw, code=code) + + +class Decrypter(Matter): + """ + Decrypter is Matter subclass with method to decrypt the plain text from a + ciper text of a fully qualified (qb64) private key/seed where private + key/seed is the plain text. Decrypter uses assymetric (public, private) key + decryption of the cipher text using its .raw as the decrypting (private) key + and its .code to indicate the cipher suite for the decryption operation. + + For example .code == MtrDex.X25519 indicates that X25519 sealed box + decyrption is used. The decryption key may be derived from an Ed25519 + signing private key that is associated with a nontransferable or basic derivation + self certifying identifier. This allows use of the self certifying identifier + to track or manage the encryption/decryption key pair. And could be used to + provide additional authentication operations for using the + encryption/decryption key pair. Support for this is provided at init time + with the sigkey parameter which allows deriving the decryption private key + from the fully qualified sigkey (signing key). + + See Matter for inherited attributes and properties: + + Attributes: + + Properties: + + + Methods: + decrypt: create cipher text + + """ + + def __init__(self, code=MtrDex.X25519_Private, seed=None, **kwa): + """ + Assign decrypting cipher suite function to ._decrypt + + Inherited Parameters: + (see Matter) + + Parameters: See Matter for inherited parameters + code (str): derivation code for private decryption key + seed (str | bytes | bytearray | memoryview | None): qb64b or qb64 + of signing key seed used to derive raw which is private + decryption key + """ + try: + super(Decrypter, self).__init__(code=code, **kwa) + except EmptyMaterialError as ex: + if seed: + signer = Signer(qb64b=seed) + if signer.code not in (MtrDex.Ed25519_Seed,): + raise ValueError("Unsupported signing seed derivation code = {}." + "".format(signer.code)) + # verkey, sigkey = pysodium.crypto_sign_seed_keypair(signer.raw) + sigkey = signer.raw + signer.verfer.raw # sigkey is raw seed + raw verkey + raw = pysodium.crypto_sign_sk_to_box_sk(sigkey) # raw private encrypt key + super(Decrypter, self).__init__(raw=raw, code=code, **kwa) + else: + raise + + if self.code == MtrDex.X25519_Private: + self._decrypt = self._x25519 + else: + raise ValueError("Unsupported decrypter code = {}.".format(self.code)) + + + def decrypt(self, *, cipher=None, qb64=None, qb2=None, klas=None, + transferable=False, bare=False, **kwa): + """Returns plain text as klas instance (Matter, Indexer, Streamer). + When klas is None then klas default is based on cipher.code or inferred + from qb64 or qb2 code. Default maybe Salter, Signer, or Streamer. + Cipher's encrypted plain text is fully qualified (qb64) + so derivaton code of plain text preserved through encryption/decryption + round trip. + + + Returns: + decrypted (Matter | Indexer | Streamer | bytes): When bare is False + returns instance of decrypted cipher text of .raw which is + encrypted qb64, qb2, or sniffable stream depending on .code + hhen Bare is True. Otherwise returns decrypted serialization + plaintext whatever that may be. + + Keyword Parameters: + (see Matter because created Decrypter is Matter subclass) + + Parameters: + cipher (Cipher): instance. One of cipher, qb64, or qb2 required. + qb64 (str | bytes | bytearray | memoryview | None ): serialization + of cipher text as fully qualified base64. When str, encodes as + utf-8. When bytearray and strip in kwa is True then strips. + qb2 (bytes | bytearray | memoryview | None ): serialization + of cipher text as fully qualified base2. Strips when bytearray + and strip in kwa is True. + klas (Matter | Indexer | Streamer): Class used to create instance from + decrypted serialization. + transferable (bool): Modifier of klas instance creation. + When klas init (such as Signer) supports transferabe parm; + True means verfer of returned signer is transferable. + False means non-transferable + bare (bool): False (default) means returns instance holding plaintext + True means returns plaintext itself + """ + if not cipher: + if qb64: # create cipher from qb64 + cipher = Cipher(qb64b=qb64, **kwa) + + elif qb2: + cipher = Cipher(qb2=qb2, **kwa) + + else: + raise EmptyMaterialError(f"Need one of cipher, qb64, or qb2.") + + return (self._decrypt(cipher=cipher, + prikey=self.raw, + klas=klas, + transferable=transferable, + bare=bare)) + + @staticmethod + def _x25519(cipher, prikey, klas=None, transferable=False, bare=False): + """ + Returns plain text as Salter or Signer instance depending on the cipher + code and the embedded encrypted plain text derivation code. + + Parameters: + cipher (Cipher): instance of encrypted seed or salt + prikey (bytes): raw binary decryption private key derived from + signing seed or sigkey + klas (Matter, Indexer, Streamer | None): Class used to create instance from + decrypted serialization. Default depends on cipher.code. + transferable (bool): Modifier of Klas instance creation. + When klas init (such as Signer) supports transferabe parm; + True means verfer of returned signer is transferable. + False means non-transferable + bare (bool): False (default) means CESR instance holding plaintext + True means plaintext + """ + # assumes raw plain text is qb64b or qb64 or sniffable stream + # so it's round trippable + pubkey = pysodium.crypto_scalarmult_curve25519_base(prikey) + plain = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) # qb64b + + if bare: + return plain + + else: + if not klas: + if cipher.code == CiXFixQB64Dex.X25519_Cipher_Salt: + klas = Salter + elif cipher.code == CiXFixQB64Dex.X25519_Cipher_Seed: + klas = Signer + elif cipher.code in CiXVarStrmDex: + klas = Streamer + else: + raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}" + f" when klas missing.") + + if cipher.code in CiXAllQB64Dex: + return klas(qb64b=plain, transferable=transferable) + elif cipher.code in CiXVarQB2Dex: + return klas(qb2=plain) + elif cipher.code in CiXVarStrmDex: + return klas(stream=plain) + else: + raise InvalidCodeError(f"Unsupported cipher code = {cipher.code}.") diff --git a/src/keri/core/streaming.py b/src/keri/core/streaming.py new file mode 100644 index 000000000..6314be482 --- /dev/null +++ b/src/keri/core/streaming.py @@ -0,0 +1,445 @@ +# -*- coding: utf-8 -*- +""" +keri.core.streaming module + +Provides support for Streamer and Annotater +""" + + +from typing import NamedTuple +from collections import namedtuple + +from .. import kering +from ..kering import sniff, Colds, Ilks + +from ..help.helping import intToB64 + + + +from .. import help + +from . import coring +from .coring import (Matter, Verser, Ilker, Diger, Prefixer, Number, Tholder, + Verfer, Traitor) + +from . import counting +from .counting import Counter + +from . import structing +from .structing import Sealer + +from . import serdering +from .serdering import Serder + + +def annot(ims): + """Annotate CESR stream + + Returns: + annotation (str): annotation of input CESR stream + + Parameters: + ims (str | bytes | bytearray | memoryview): CESR incoming message stream + as qb64 (maybe qb2) + + """ + oms = bytearray() + indent = 0 + + + if not isinstance(ims, bytearray): # going to strip + ims = bytearray(ims) # so make bytearray copy, converts str to bytearray + + while ims: # right now just for KERI event messages + cold = sniff(ims) # check for spurious counters at front of stream + + if cold in Colds.txt: + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # Key Event Counter " + f"{val.name} count={val.count} quadlets\n".encode()) + indent += 1 + + # should grab fill message frame here so can verify its consumed + # when done annotating + + # version + val = Verser(qb64b=ims, strip=True) + versage = val.versage + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'v' version Verser {val.name} " + f"proto={versage.proto} vrsn={versage.vrsn.major}." + f"{versage.vrsn.minor:02}\n".encode()) + # ilk + ilker = Ilker(qb64b=ims, strip=True) + ilk = ilker.ilk + oms.extend(f"{' ' * indent * 2}{ilker.qb64} # 't' message type Ilker " + f"{ilker.name} Ilk={ilker.ilk}\n".encode()) + + + + if ilk in (Ilks.icp, Ilks.ixn, Ilks.rot, Ilks.dip, Ilks.drt): + # said + val = Diger(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'd' SAID Diger " + f"{val.name} \n".encode()) + + # aid pre + val = Prefixer(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'i' AID Prefixer " + f"{val.name} \n".encode()) + + # sn + val = Number(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 's' Number " + f"{val.name} sn={val.sn}\n".encode()) + + if ilk in (Ilks.icp, Ilks.dip): # inception + # v='', t='', d='', i='', s='0', kt='0',k=[], nt='0', n=[], bt='0', b=[], c=[], a=[] + + # Signing key threshold + val = Tholder(limen=ims, strip=True) # add qb64 and qb2 to tholder as aliases for limen and limen.decode() + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'kt' " + f"Tholder signing threshold={val.sith}\n".encode()) + # Signing key list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'k' Signing Key " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Verfer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key Verfer " + f"{val.name}\n".encode()) + indent -= 1 + # Rotation key threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'nt' " + f"Tholder rotation threshold={val.sith}\n".encode()) + # Next key digest list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'n' Rotation Key " + f"Digest List Counter {val.name} count={val.count} " + f"quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Diger(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key digest Diger" + f" {val.name}\n".encode()) + indent -= 1 + # Witness Backer threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'bt' " + f"Tholder Backer (witness) threshold={val.sith}\n".encode()) + # Witness Backer list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'b' Backer (witness)" + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Prefixer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # AID Prefixer " + f"{val.name}\n".encode()) + indent -= 1 + # Config Trait List + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'c' Config Trait " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Traitor(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # trait Traitor " + f"{val.name} trait={val.trait}\n".encode()) + indent -= 1 + + + elif ilk == Ilks.ixn: # Interaction + # v='', t='',d='', i='', s='0', p='', a=[] + # Prior said + val = Diger(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'p' prior SAID Diger " + f"{val.name} \n".encode()) + + + + elif ilk in (Ilks.rot, Ilks.drt): # Rotation + # v='', t='',d='', i='', s='0', p='', kt='0',k=[], nt='0', n=[], bt='0', br=[], ba=[], c=[], a=[] + # Prior said + val = Diger(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'p' prior SAID Diger " + f"{val.name} \n".encode()) + + # Signing key threshold + val = Tholder(limen=ims, strip=True) # add qb64 and qb2 to tholder as aliases for limen and limen.decode() + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'kt' " + f"Tholder signing threshold={val.sith}\n".encode()) + # Signing key list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'k' Signing Key " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Verfer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key Verfer " + f"{val.name}\n".encode()) + indent -= 1 + # Rotation key threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'nt' " + f"Tholder rotation threshold={val.sith}\n".encode()) + # Next key digest list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'n' Rotation Key " + f"Digest List Counter {val.name} count={val.count} " + f"quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Diger(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # key digest Diger" + f" {val.name}\n".encode()) + indent -= 1 + # Witness Backer threshold + val = Tholder(limen=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.limen.decode()} # 'bt' " + f"Tholder Backer (witness) threshold={val.sith}\n".encode()) + + # Witness Backer Cut list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'b' Backer (witness)" + f"Cut List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Prefixer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # AID Prefixer " + f"{val.name}\n".encode()) + indent -= 1 + + # Witness Backer Add list + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'b' Backer (witness)" + f"Add List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Prefixer(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # AID Prefixer " + f"{val.name}\n".encode()) + indent -= 1 + + # Config Trait List + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'c' Config Trait " + f"List Counter {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Traitor(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # trait Traitor " + f"{val.name} trait={val.trait}\n".encode()) + indent -= 1 + + + else: + raise kering.IlkError(f"Unexpected message type={ilk}") + + if ilk in (Ilks.icp, Ilks.ixn, Ilks.rot, Ilks.dip, Ilks.drt): + # Seal (anchor) List + val = Counter(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'a' Seal List Counter" + f" {val.name} count={val.count} quadlets\n".encode()) + indent += 1 + frame = ims[:val.count*4] # extract frame from ims + del ims[:val.count*4] # strip frame + while frame: + val = Counter(qb64b=frame, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # Seal Counter " + f"{val.name} count={val.count} quadlets\n".encode()) + indent += 1 + subframe = frame[:val.count*4] + del frame[:val.count*4] # strip subframe + clan = Sealer.Clans[Serder.CodeClans[val.code]] + while subframe: + val = Sealer(clan=clan, qb64=subframe, strip=True) # need to add qb64 parameter to structor + oms.extend(f"{' ' * indent * 2}{val.qb64}# seal Sealer {val.name}\n".encode()) + indent += 1 + for i, t in enumerate(val.crew._asdict().items()): + oms.extend(f"{' ' * indent * 2}# '{t[0]}' = " + f"{t[1]}\n".encode()) + indent -= 1 + indent -= 1 + indent -= 1 + + if ilk == Ilks.dip: + # delegator aid delpre + val = Prefixer(qb64b=ims, strip=True) + oms.extend(f"{' ' * indent * 2}{val.qb64} # 'di' Delegator AID Prefixer " + f"{val.name} \n".encode()) + + if ims: + raise kering.ExtractionError(f"Unexpected remaining bytes in stream.") + + elif cold in Colds.bny: + pass + + else: + raise kering.ColdStartError("Expecting stream tritet={}" + "".format(cold)) + + return oms.decode() # return unicode string + +def denot(ams): + """De-annotate CESR stream + + Returns: + dms (bytes): deannotation of input annotated CESR message stream + + Parameters: + ams (str): CESR annotated message stream text + """ + dms = bytearray() # deannotated message stream + lines = ams.splitlines() + for line in lines: + line = line.strip() + front, sep, back = line.partition('#') # finde comment if any + front = front.strip() # non-commented portion strip white space + if front: + dms.extend(front.encode()) + + return bytes(dms) + + +class Streamer: + """ + Streamer is CESR sniffable stream class + + + Has the following public properties: + + Properties: + stream (bytearray): sniffable CESR stream + + + Methods: + + + Hidden: + _verify() -> bool + + + + """ + + def __init__(self, stream, verify=False): + """Initialize instance + Holds sniffable CESR stream as byte like string + either (bytes, bytearray, or memoryview) + + + Parameters: + stream (str | bytes | bytearray | memoryview): sniffable CESR stream + verify (bool): When True raise error if .stream is not sniffable. + + + """ + if hasattr(stream, "encode"): + stream = bytearray(stream.encode()) # convert str to bytearray + if not isinstance(stream, (bytes, bytearray, memoryview)): + raise kering.InvalidTypeError(f"Invalid stream type, not byteable.") + + self._stream = stream + + + def _verify(self): + """Returns True if .stream is sniffable, False otherwise + + Returns: + sniffable (bool): True when .stream is sniffable. + False otherwise. + Only works for ver 2 CESR because need for all count codes to be + pipelineable in order to simply parse stream + + """ + return False + + + @property + def stream(self): + """stream property getter + """ + return self._stream + + + @property + def text(self): + """expanded stream where all primitives and groups in stream are + individually expanded to qb64. + Requires parsing full depth to ensure expanded consistently. + Returns: + stream (bytes): expanded text qb64 version of stream + + Only works for ver 2 CESR because need for all count codes to be + pipelineable in order to simply parse and expand stream + + """ + return self._stream + + @property + def binary(self): + """compacted stream where all primitives and groups in stream are + individually compacted to qb2. + Requires parsing full depth to ensure compacted consistently + Returns: + stream (bytes): compacted binary qb2 version of stream + + Only works for ver 2 CESR because need for all count codes to be + pipelineable in order to simply parse and compact stream + + """ + return self._stream + + @property + def texter(self): + """stream as Texter instance. + Texter(text=self.stream) + Returns: + texter (Texter): Texter primitive of stream suitable wrapping + + """ + return self._stream + + @property + def bexter(self): + """stream as Bexter instance. + Bexter of expanded text version of stream. + First expand to text which requires parsing then create bexter + Bexter(bext=self.text) + Because sniffable stream MUST NOT start with 'A' then there is no + length ambiguity. The only tritet collison of 'A' is with '-' but the + remaining 5 bits are guaranteed to always be different. So bexter must + check not just the starting tritet but the full starting byte to ensure + not 'A' as first byte. + + Requires parsing to ensure qb64 + Returns: + bexter (Bexter): Bexter primitive of stream suitable wrapping + + """ + return self._stream + + + + + diff --git a/src/keri/core/structing.py b/src/keri/core/structing.py new file mode 100644 index 000000000..a9ef531f4 --- /dev/null +++ b/src/keri/core/structing.py @@ -0,0 +1,636 @@ +# -*- coding: utf-8 -*- +""" +keri.core.indexing module + +Provides versioning support for Indexer classes and codes +""" + + +from typing import NamedTuple +from collections import namedtuple +from collections.abc import Mapping +from dataclasses import dataclass, astuple, asdict + +from ..kering import (EmptyMaterialError, InvalidValueError,) + +from .. import help +from ..help import nonStringSequence + +from . import coring +from .coring import (MapDom, Matter, Diger, Prefixer, Number) + + + +# ToDo Change seal namedtuple definitions to NamedTuple subclasses so can +# use typehints on field values which type hints are the primitive types. Use +# union | on type hints to allow qb64, qb2, primitive instance, primitive class +# as acceptable values This provides more clarity in documentation. Actually +# enforcing types is harder with union | but can still be accomplished. + +# for the following Seal namedtuples use the ._asdict() method to convert to dict +# when using in events +# to convert seal namedtuple to dict use namedtuple._asdict() +# seal == SealEvent(i="abc",s="1",d="efg") +# sealdict =seal._asdict() +# to convet dict to namedtuple use ** unpacking as in seal = SealDigest(**sealdict) +# to check if dict of seal matches fields of associted namedtuple +# if tuple(sealdict) == SealEvent._fields: + +# Digest Seal: uniple (d,) +# d = digest qb64 of data (usually SAID) +SealDigest = namedtuple("SealDigest", 'd') + +# Root Seal: uniple (rd,) +# rd = Merkle tree root digest qb64 digest of anchored (sealed) data in Merkle tree +SealRoot = namedtuple("SealRoot", 'rd') + +# Backer Seal: couple (bi, d) +# bi = pre qb64 backer nontrans identifier prefix +# d = digest qb64 of backer metadata anchored to event usually SAID of data +SealBacker = namedtuple("SealBacker", 'bi d') + +# Last Establishment Event Seal: uniple (i,) +# i = pre is qb64 of identifier prefix of KEL from which to get last est, event +# used to indicate to get the latest keys available from KEL for 'i' +SealLast = namedtuple("SealLast", 'i') + +# Transaction Event Seal for Transaction Event: duple (s, d) +# s = sn of transaction event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of transaction event +# the pre is provided in the 'i' field qb64 of identifier prefix of KEL +# key event that this seal appears. +# use SealSourceCouples count code for attachment +SealTrans = namedtuple("SealTrans", 's d') + +# Event Seal: triple (i, s, d) +# i = pre is qb64 of identifier prefix of KEL for event, +# s = sn of event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of event +SealEvent = namedtuple("SealEvent", 'i s d') + + +# Following are not seals only used in database + +# State Establishment Event (latest current) : quadruple (s, d, br, ba) +# s = sn of latest est event as lowercase hex string no leading zeros, +# d = SAID digest qb64 of latest establishment event +# br = backer (witness) remove list (cuts) from latest est event +# ba = backer (witness) add list (adds) from latest est event +StateEstEvent = namedtuple("StateEstEvent", 's d br ba') + +# not used should this be depricated? +# State Event (latest current) : triple (s, t, d) +# s = sn of latest event as lowercase hex string no leading zeros, +# t = message type of latest event (ilk) +# d = SAID digest qb64 of latest event +StateEvent = namedtuple("StateEvent", 's t d') + + +# Cast conversion: duple (c, k) +# kls = primitive class reference in order to cast as appropriate +# namedtuple with values as primitive classes +# prm = primitive __init__ keyword parameter name to use when casting +# default None. When default then use qb64 or qb64b as appropriate. +Castage = namedtuple('Castage', "kls prm", defaults=(None, )) + + +@dataclass(frozen=True) +class EmptyClanDom(MapDom): + """ + SealClanDom is dataclass of namedtuple seal class references (clans) each + indexed by its class name. + + Only provide defined classes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get class reference with item syntax using + name variables. + + Example: EmptyClanDex[name] + """ + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +EClanDom = EmptyClanDom() # create instance + + +@dataclass(frozen=True) +class EmptyCastDom(MapDom): + """ + SealCastCodex is dataclass of namedtuple instances (seal casts) whose values + are named primitive class references + + indexed by its namedtuple class name. + + Only provide defined namedtuples casts. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get namedtuple instance with item syntax using + name variables. + + Example: EmptyCastDex[name] + """ + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +ECastDom = EmptyCastDom() # create instance + + +@dataclass(frozen=True) +class SealClanDom(MapDom): + """ + SealClanDom is dataclass of namedtuple seal class references (clans) each + indexed by its class name. + + Only provide defined classes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get class reference with item syntax using + name variables. + + Example: ClanDom[name] + """ + SealDigest: type[NamedTuple] = SealDigest # SealDigest class reference + SealRoot: type[NamedTuple] = SealRoot # SealRoot class reference + SealBacker: type[NamedTuple] = SealBacker # SealBacker class reference + SealLast: type[NamedTuple] = SealLast # SealLast class reference single + SealTrans: type[NamedTuple] = SealTrans # SealTrans class reference couple + SealEvent: type[NamedTuple] = SealEvent # SealEvent class reference triple + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +SClanDom = SealClanDom() # create instance + + + + +@dataclass(frozen=True) +class SealCastDom(MapDom): + """ + SealCastDom is dataclass of namedtuple instances (seal casts) whose values + are named primitive class references + + indexed by its namedtuple class name. + + Only provide defined namedtuples casts. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + As subclass of MapCodex can get namedtuple instance with item syntax using + name variables. + + Example: CastDom[name] + """ + SealDigest: NamedTuple = SealDigest(d=Castage(Diger)) # SealDigest class reference + SealRoot: NamedTuple = SealRoot(rd=Castage(Diger)) # SealRoot class reference + SealBacker: NamedTuple = SealBacker(bi=Castage(Prefixer), + d=Castage(Diger)) # SealBacker class reference + SealLast: NamedTuple = SealLast(i=Castage(Prefixer)) # SealLast class reference single + SealTrans: NamedTuple = SealTrans(s=Castage(Number, 'numh'), + d=Castage(Diger)) # SealTrans class reference couple + SealEvent: NamedTuple = SealEvent(i=Castage(Prefixer), + s=Castage(Number, 'numh'), + d=Castage(Diger)) # SealEvent class reference triple + + def __iter__(self): + return iter(astuple(self)) # enables value not key inclusion test with "in" + +SCastDom = SealCastDom() # create instance + + + +class Structor: + """Structor class each instance holds a namedtuple .data of named values. + Each value is a primitive instance of CESR primitive subclass that supports + text (ab64) and binary (qb2) domains. + Structor instances can be serialized to or deserialized from concatenation + of the qb64 or qb2 representations of the data values. Creation requires + input of a ordered named classes for creating the named instances from the + input data. Smart data format input are supported to accomodate the many ways + the named data may appear in messages and or databases. + + Instance Creation Patterns: + + Structor(data): + + Structor(clan, cast, crew): + Structor(clan, cast, qb64): + Structor(clan, cast, qb2): + + Structor(cast, crew): + Structor(cast, qb64): + Structor(cast, qb2): + + Structor(clan, crew): when known cast in .Casts for clan + Structor(clan, qb64): when known cast in .Casts for clan + Structor(clan, qb2): when known cast in .Casts for clan + + Structor(crew): when known cast in .Casts for crew + + + Class Attributes: + Clans (type[Namedtuple]): each value is known NamedTuple class keyed + by its own field names (tuple). Enables easy query of its values() to + find known data types given field names tuple. + + Casts (NamedTuple): each value is primitive class of cast keyed by fields + names of the associated NamedTuple class in .Clans. Enables finding + known primitive classes given NamedTuple class of clan or instance + of cast or crew. + + When known casts or provided in .Clans/.Casts then more flexible creation + is supported for different types of provided cast and crew. + When no clan is provided and an unknown cast and/or crew are provided as + Mappings then Structor may create custom clan from the names given by the + cast and/or crew keys(). Subclasses may override this behavior by raising + an exception for unknown or custom clans. + + + Properties: + data (NamedTuple): fields are named instances of CESR primitives + clan (type[NamedTuple]): class reference of .data's class + cast (NamedTuple | None): values are Castage instances that each provide + CESR primitive class references and primitive init parameter + used to .data's primitive instances. + crew (NamedTuple): named qb64 values of .data's primitive instances + qb64 (str): concatenated data values as qb64 str of data's primitives + qb64b (bytes): concatenated data values as qb64b of data's primitives + qb2 (bytes): concatenated data values as qb2 bytes of data's primitives + + + Methods: + + + Hidden: + _data (NamedTuple): named CESR primitive instances + + + """ + Clans = EClanDom # known namedtuple clans. Override in subclass with non-empty + Casts = ECastDom # known namedtuple casts. Override in subclass with non-empty + # Create .Names dict that maps tuple of clan/cast fields names to its namedtuple + # class type name so can look up a know clan or cast given a matching tuple + # of either field names from a namedtuple or keys from a dict. The tuple of + # field names is a mark of the structor type. Maps mark to class name. + Names = {tuple(clan._fields): clan.__name__ for clan in Clans} + + + def __init__(self, data=None, *, clan=None, cast=None, crew=None, + qb64=None, qb2=None, strip=False): + """Initialize instance + + Parameters: + data (NamedTuple | None): fields are named primitive instances for .data + Given data can derive clan, cast, crew, qb64, and qb2 + clan (type[NamedTuple]): provides class reference for generated .data + when data missing. + cast (NamedTuple | dict | Iterable | None): values are Castage + instances that each provide CESR primitive class references + and primitive init parameter used to .data's primitive + instances. None means .data provided directly not generated + from cast + each value provides CESR + primitive subclass reference used to create primitive instances + for generating .data. Can be used to infer namedtuple type of + .data when data and clan missing. Takes precendence over crew. + crew (NamedTuple | dict | Iterable | None): each value provides qb64 value + of primitive for generating .data with .cast when data missing. + Can be used to infer namedtuple type of .data when data and clan + missing. + qb64 (str | bytes | bytearray | None): concatenation of qb64 data values to + generate .data with data and crew missing. + qb2 (bytes | bytearray | None): concatenation of qb2 data values to generate + .data when data and crew and qb64 missing. + strip (bool): False means do not strip each value from qb64 or qb2. + Default is False. + True means if qb64 or qb2 are bytearray then strip + contained concatenated data values. Enables parser + to extract data fields from front of CESR stream. + + + """ + if data: + if not (isinstance(data, tuple) and hasattr(data, "_fields")): + raise InvalidValueError(f"Not namedtuple subclass {data=}.") + + for pi in data: # check for primitive interface + if not (hasattr(pi, "qb64") and hasattr(pi, "qb2")): + raise InvalidValueError(f"Non-primitive data member={pi}.") + + cast = None # ensure cast is None since not used to generate data + + + else: + if not clan: # attempt to get from cast and/or crew + if cast and isinstance(cast, tuple) and hasattr(cast, "_fields"): + clan = cast.__class__ + + if not clan and crew: + if isinstance(crew, tuple) and hasattr(crew, "_fields"): + clan = crew.__class__ + + if not clan and isinstance(cast, Mapping): # get clan from cast + mark = tuple(cast) # create custom clan based on cast mark + if (cname := self.Names.get(mark)): # get known else None + clan = self.Clans[cname] + cast = self.Casts[cname] + + else: # create custom clan from cast + clan = namedtuple("_".join(mark), mark) # custom clan + + if not clan and isinstance(crew, Mapping): # get clan from crew + mark = tuple(crew) # create custom clan based on crew mark + if (cname := self.Names.get(mark)): # get known else None + clan = self.Clans[cname] + cast = self.Casts[cname] + + else: # create custom clan from crew + clan = namedtuple("_".join(mark), mark) # custom clan from cast keys + + if clan: + if not (issubclass(clan, tuple) and hasattr(clan, "_fields")): + raise InvalidValueError(f"Not namedtuple subclass {clan=}.") + else: + raise InvalidValueError(f"Missing or unobtainable clan.") + + # have clan but may not have cast + if cast: + if not isinstance(cast, clan): + if isinstance(cast, tuple) and hasattr(cast, "_fields"): + if cast._fields != clan._fields: # fields is mark + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and cast=" + f"{cast._fields}.") + + cast = clan(**cast._asdict()) # convert to clan + + elif isinstance(cast, Mapping): + if tuple(cast) != clan._fields: # fields is mark + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and keys cast=" + f"{tuple(cast)}.") + + cast = clan(**cast) # convert to clan + + elif isinstance(cast, nonStringSequence): + cast = clan(*cast) # convert to clan assumes elements in correct order + + else: + raise InvalidValueError(f"Invalid {cast=}.") + + else: # get cast from known .Casts if possible + if (cname := self.Names.get(clan._fields)): # fields is mark + cast = self.Casts[cname] # get known cast + else: # cast missing or unobtainable + raise InvalidValueError(f"Missing or unobtainable cast.") + + # have cast now + for cstg in cast: + if not (hasattr(cstg.kls, "qb64") and hasattr(cstg.kls, "qb2")): + raise InvalidValueError(f"Cast member {cstg.kls=} not CESR" + " Primitive.") + + # have clan and cast but may not have crew + if crew: + if not isinstance(crew, clan): + if isinstance(crew, tuple) and hasattr(crew, "_fields"): + if crew._fields != clan._fields: # fields is mark + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and crew=" + f"{crew._fields}.") + + crew = clan(**crew._asdict()) # convert to clan + + elif isinstance(crew, Mapping): + if tuple(crew) != clan._fields: # fields is mark + raise InvalidValueError(f"Mismatching fields clan=" + f"{clan._fields} and keys crew=" + f"{tuple(crew)}.") + + crew = clan(**crew) # convert to clan + + elif isinstance(crew, nonStringSequence): + crew = clan(*crew) # convert to clan assumes elements in correct order + + else: + raise InvalidValueError(f"Invalid {crew=}.") + + data = clan(*(cstg.kls(**{cstg.prm if cstg.prm is not None else 'qb64': val}) + for cstg, val in zip(cast, crew))) + # data = clan(*(klas(qb64=val) for klas, val in zip(cast, crew))) + + elif qb64: + if hasattr(qb64, "encode"): + qb64 = qb64.encode() + + if strip: + if not isinstance(qb64, bytearray): + qb64 = bytearray(qb64) + + data = clan(*(cstg.kls(qb64b=qb64, strip=strip) for cstg in cast)) + + else: + o = 0 # offset into memoryview of qb64 + pis = [] # primitive instances + mv = memoryview(qb64) + for cstg in cast: # Castage + pi = cstg.kls(qb64b=mv[o:]) + pis.append(pi) + o += len(pi.qb64b) + data = clan(*pis) + + elif qb2: + if strip: + if not isinstance(qb2, bytearray): + qb2 = bytearray(qb2) + + data = clan(*(cstg.kls(qb2=qb2, strip=strip) for cstg in cast)) + + else: + o = 0 # offset into memoryview of qb2 + pis = [] # primitive instances + mv = memoryview(qb2) + for cstg in cast: # Castage + pi = cstg.kls(qb2=mv[o:]) + pis.append(pi) + o += len(pi.qb2) + data = clan(*pis) + + else: + raise EmptyMaterialError("Need crew or qb64 or qb2.") + + + self._data = data + self._cast = (cast if cast is not None else + self.clan(*(Castage(val.__class__) for val in self.data))) + + + + + @property + def data(self): + """Returns: + data (NamedTuple): ._data namedtuple of primitive instances + + Getter for ._data makes it read only + """ + return self._data + + + @property + def clan(self): + """Returns: + clan (type[NamedTuple]): class of .data + + """ + return self.data.__class__ + + @property + def name(self): + """Returns: + name (str): name of class of .data + + """ + return self.data.__class__.__name__ + + @property + def cast(self): + """Return: + cast (NamedTuple): named primitive classes in .data + + Getter for ._cast makes it read only when not None + """ + return self._cast + + @property + def crew(self): + """Return: + crew (NamedTuple): named qb64 field values from .data + + """ + return (self.clan(*(getattr(val, cstg.prm if cstg.prm is not None else "qb64") + for cstg, val in zip(self.cast, self.data)))) + # return self.clan(*(val.qb64 for val in self.data)) + + + @property + def asdict(self): + """Returns: + map (dict): .data as a dictionary + + """ + return self.data._asdict() + + + @property + def qb64(self): + """Returns: + qb64 (str): concatenated qb64 of each primitive in .data + """ + return (''.join(val.qb64 for val in self.data)) + + + @property + def qb64b(self): + """Returns: + qb64b (bytes): concatenated qb64b of each primitive in .data + """ + return (b''.join(val.qb64b for val in self.data)) + + + @property + def qb2(self): + """Returns: + qb2 (bytes): concatenated qb2 of each primitive in .data + + """ + return (b''.join(val.qb2 for val in self.data)) + + + + +class Sealer(Structor): + """Sealer is Structor subclass each instance holds a namedtuple .data of + named values belonging to KERI Seals for anchoring in messages or adding + to message attachments. + + See Structor class for more details. + + + Inherited Class Attributes: + Clans (type[Namedtuple]): each value is known NamedTuple class keyed + by its own field names (tuple). Enables easy query of its values() to + find known data types given field names tuple. + + Casts (NamedTuple): each value is primitive class of cast keyed by fields + names of the associated NamedTuple class in .Clans. Enables finding + known primitive classes given NamedTuple class of clan or instance + of cast or crew. + + When known casts or provided in .Clans/.Casts then more flexible creation + is supported for different types of provided cast and crew. + When no clan is provided and an unknown cast and/or crew are provided as + Mappings then Structor may create custom clan from the names given by the + cast and/or crew keys(). Subclasses may override this behavior by raising + an exception for unknown or custom clans. + + + Inherited Properties: + data (NamedTuple): fields are named instances of CESR primitives + clan (type[NamedTuple]): class reference of .data's class + cast (NamedTuple): CESR primitive class references of .data's primitive + instances + crew (NamedTuple): named qb64 values of .data's primitive instances + qb64 (str): concatenated data values as qb64 str of data's primitives + qb64b (bytes): concatenated data values as qb64b of data's primitives + qb2 (bytes): concatenated data values as qb2 bytes of data's primitives + + + Methods: + + + Hidden: + _data (NamedTuple): named CESR primitive instances + + + """ + Clans = SClanDom # known namedtuple clans. Override in subclass with non-empty + Casts = SCastDom # known namedtuple casts. Override in subclass with non-empty + # Create .Names dict that maps clan/cast fields names to its namedtuple + # class type name so can look up a know clan or cast given a matching set + # of either field names from a namedtuple or keys from a dict. + Names = {tuple(clan._fields): clan.__name__ for clan in Clans} + + + def __init__(self, *pa, **kwa): + """Initialize instance + + + Inherited Parameters: + data (NamedTuple): fields are named primitive instances for .data + Given data can derive clan, cast, crew, qb64, and qb2 + clan (type[NamedTuple]): provides class reference for generated .data + when data missing. + cast (NamedTuple | dict | Iterable): each value provides CESR + primitive subclass reference used to create primitive instances + for generating .data. Can be used to infer namedtuple type of + .data when data and clan missing. Takes precendence over crew. + crew (NamedTuple | dict | Iterable): each value provides qb64 value + of primitive for generating .data with .cast when data missing. + Can be used to infer namedtuple type of .data when data and clan + missing. + qb64 (str | bytes | bytearray): concatenation of qb64 data values to + generate .data with data and crew missing. + qb2 (bytes | bytearray): concatenation of qb2 data values to generate + .data when data and crew and qb64 missing. + strip (bool): False means do not strip each value from qb64 or qb2. + Default is False. + True means if qb64 or qb2 are bytearray then strip + contained concatenated data values. Enables parser + to extract data fields from front of CESR stream. + + """ + + super(Sealer, self).__init__(*pa, **kwa) diff --git a/src/keri/db/basing.py b/src/keri/db/basing.py index b4f0f095d..58db47790 100644 --- a/src/keri/db/basing.py +++ b/src/keri/db/basing.py @@ -37,10 +37,9 @@ import keri from . import dbing, koming, subing -from .dbing import splitKey, dgKey from .. import kering - -from ..core import coring, eventing, parsing, serdering +from .. import core +from ..core import coring, eventing, parsing, serdering, indexing from .. import help from ..help import helping @@ -48,12 +47,40 @@ logger = help.ogler.getLogger() + MIGRATIONS = [ ("0.6.8", ["hab_data_rename"]), - ("1.0.0", ["add_key_and_reg_state_schemas"]) + ("1.0.0", ["add_key_and_reg_state_schemas"]), + ("1.2.0", ["rekey_habs"]) ] +# ToDo XXXX maybe +''' +class komerdict(dict): + """ + Subclass of dict that has db as attribute and employs read through cache + from db Baser.stts of kever states to reload kever from state in database + when not found in memory as dict item. + + add method that answers is a given pre a group hab pre .localGroup(pre) + + ToDo XXXX change name of dbdict to stateDict since now have differen types + and can't subclass dict with init parameters. + but can change function by manually assigning attributes but that is ugly + need wrapper decorator to do that. So can update attributes with wrapper + on class that injects instance attributes when class is instanced + one of the injected parameters is function that that maps returned Komer to + object class + parameters are subdb (must be Komer) and function that maps retrieved dataclass + record from dataabase to class instance. if no mapping function then just + return the dataclass record as value. + """ + +''' + + +# ToDo XXXX change name to statedict since not a generic dbdict class dbdict(dict): """ Subclass of dict that has db as attribute and employs read through cache @@ -108,6 +135,9 @@ def get(self, k, default=None): return self.__getitem__(k) + + + @dataclass class RawRecord: """RawRecord is base class for dataclasses that provides private utility @@ -223,6 +253,18 @@ class KeyStateRecord(RawRecord): # baser.state di: str = '' # delegator aid qb64 if any otherwise empty '' str +@dataclass +class EventSourceRecord: # tracks source of event local or remote + """ + Keyed by dig (said) of serder of event + + Usage: + + """ + local: bool = True # True of local (protected) else False for remote (unprotected) + + def __iter__(self): + return iter(asdict(self)) @dataclass @@ -240,6 +282,8 @@ class HabitatRecord: # baser.habs """ hid: str # hab own identifier prefix qb64 + name: str | None = None + domain: str | None = None mid: str | None = None # group member identifier qb64 when hid is group smids: list | None = None # group signing member ids when hid is group rmids: list | None = None # group rotating member ids when hid is group @@ -279,7 +323,7 @@ class OobiQueryRecord: # information for responding to OOBI query constraint policy for endpoint discovery . Usage: - oobiqs: dict[str, OobiQueryRecord] = field(default_factory=dict) + """ cid: str = None # qb64 role: str = None # one of kering.Roles None is any or all @@ -305,7 +349,6 @@ class OobiRecord: urls: list = None - @dataclass class EndpointRecord: # baser.ends """ @@ -457,6 +500,56 @@ def __iter__(self): return iter(asdict(self)) +@dataclass +class ObservedRecord: # baser.obvs + """ + Watched Record with fields and keys to manage OIDs (Observed IDs) being watched by a watcher, keyed by + cid (controller ID), aid (watcher ID), and oid (observed ID). + + The namespace is a tree of branches with each leaf at a + specific (cid, aid, oid). Retrieval by branch returns groups of leaves as + appropriate for a cid braanch or cid.aid branch. + Database Keys are (cid, aid, oid) where cid is attributable controller identifier + (qb64 prefix). + + Attributes: + enabled (bool): AuthZ via expose message + True means oid is enabled as being observed + False means eid is disenabled being observed + None means eid is neither enabled or disenabled + name (str): user friendly name for eid in role + datetime (str): Date time this record was last observed + + + A watcher end reply message is required from which the field values + for this record are extracted. A routes of /watcher/{aid}/add /watcher/{aid}/cut + Uses add-cut model with allowed field + enabled==True oid is allowed (add) as being observed + enabled==False oid is disallowed (cut) as being observed + + { + "v" : "KERI10JSON00011c_", + "t" : "rpy", + "d": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "dt": "2020-08-22T17:50:12.988921+00:00", + "r" : "/watcher/BrHLayDN-mXKv62DAjFLX1_Y5yEUe0vA9YPe_ihiKYHE/add", + "a" : + { + "cid": "EaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM", + "oid": "EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + "oobi": "http://example.com/oobi/EyX-zd8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM", + } + } + + """ + enabled: bool = None # True eid enabled (add), False eid disenabled (cut), None neither + name: str = "" # optional user friendly name of endpoint + datetime: str = None + + def __iter__(self): + return iter(asdict(self)) + + @dataclass class WellKnownAuthN: """ @@ -501,7 +594,7 @@ def reopenDB(db, clear=False, **kwa): finally: db.close(clear=clear) - +# Env var for configuring LMDB size for the main Baser database KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" @@ -515,20 +608,83 @@ class Baser(dbing.LMDBer): kevers (dict): Kever instances indexed by identifier prefix qb64 prefixes (OrderedSet): local prefixes corresponding to habitats for this db - .evts is named sub DB whose values are serialized events + .evts is named sub DB whose values are serialized key events dgKey DB is keyed by identifier prefix plus digest of serialized event Only one value per DB key is allowed - .fels is named sub DB of first seen event log table (FEL) of digests - that indexes events in first 'seen' accepted order for replay and - cloning of event log. Only one value per DB key is allowed. - Provides append only ordering of accepted first seen events. + .kels is named sub DB of key event logs as indices that map sequence numbers + to serialized key event digests. + Actual serialized key events are stored in .evts by SAID digest + Uses sequence number or sn. + snKey + Values are digests used to lookup event in .evts sub DB + DB is keyed by identifier prefix plus sequence number of key event + More than one value per DB key is allowed + + .fels is named sub DB of first seen event logs (FEL) as indices that map + first seen ordinal number to digests. + Actual serialized key events are stored in .evts by SAID digest + This indexes events in first 'seen' accepted order for replay and + cloning of event log. Uses first seen order number or fn. fnKey DB is keyed by identifier prefix plus monotonically increasing first seen order number fn. Value is digest of serialized event used to lookup event in .evts sub DB + Only one value per DB key is allowed. + Provides append only ordering of accepted first seen events. + + .fons is named subDB CesrSuber + Uses digest + dgKey + Maps prefix and digest to fn value (first seen ordinal number) of + the associated event. So one used pre and event digest, get its fn here + and then use fn to fetch event from .evnts by fn from .fels. + This ensures that any event looked up this way was first seen at + some point in time even if later superseded by a recovery rotation. + Whereas direct lookup in .evts could be escrowed events that may + never have been accepted as first seen. + CesrSuber(db=self, subkey='fons.', klas=core.Number) + + .esrs is named sub DB instance of Komer of EventSourceRecord + dgKey + DB is keyed by identifier prefix plus digest (said) of serialized event + Value is serialized instance of EventSourceRecord dataclass. + Only one value per DB key is allowed. + Keeps track of the source of the event. When .local is Truthy the + event was sourced in a protected way such as being generated + locally or via a protected path. When .local is Falsey the event was + NOT sourced in a protected way. The value of .local determines what + validation logic to run on the event. This database is used to track + the source when processing escrows that would otherwise be decoupled + from the original source of the event. + + .misfits is named sub DB instance of CesrIoSetSuber for misfit escrows + subkey "mfes." + snKey + DB is keyed by event controller prefix plus sn of serialized event + where sn is 32 char hex string with leading zeros + Value is serialized qb64b dig (said) of event + Misfit escrows are events with remote (nonlocal) sources that are + inappropriate (i.e. would be dropped) unless they can be promoted + to local source via some extra after the fact authentication. + Escrow processing determines if and how to promote event source to + local and then reprocess + + .delegables is named sub DB instance of CesrIoSetSuber for delegable event + escrows of key event with local delegator that need approval. + subkey "dees." delegable event escrows + snKey + DB is keyed by event controller prefix plus sn of serialized event + where sn is 32 char hex string with leading zeros + Value is serialized qb64b dig (said) of event + Delegable event escrows are events with local delegator that need + to be approved via the anchoring of the delegated event seal in + the delegator's KEL. Event source must be local. A nonlocal (remote) + source for a delegable event of a local delegator must first pass + through the misfit escrow and get promoted to local source. + .dtss is named sub DB of datetime stamp strings in ISO 8601 format of the datetime when the event was first escrosed and then later first @@ -553,8 +709,9 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .wigs is named sub DB of indexed witness signatures of event - Witnesses always have nontransferable indetifier prefixes. + .wigs is named sub DB of indexed witness signatures of event that may + come directly or derived from a witness receipt message. + Witnesses always have nontransferable identifier prefixes. The index is the offset of the witness into the witness list of the most recent establishment event wrt the receipted event. dgKey @@ -562,7 +719,10 @@ class Baser(dbing.LMDBer): More than one value per DB key is allowed .rcts is named sub DB of event receipt couplets from nontransferable - signers. Each couple is concatenation of fully qualified items. + signers. + These are endorsements from nontrasferable signers who are not witnesses + May be watchers or other + Each couple is concatenation of fully qualified items. These are: non-transferale prefix plus non-indexed event signature by that prefix. dgKey @@ -574,6 +734,8 @@ class Baser(dbing.LMDBer): qualified items. These are: receipted event digest, non-transferable receiptor identifier prefix, plus nonindexed receipt event signature by that prefix. + Used to manage out of order events such as escrowing + receipt couple until event receipted shows up. snKey DB is keyed by receipted event controller prefix plus sn of serialized event @@ -584,6 +746,8 @@ class Baser(dbing.LMDBer): of validator. These are: transferable prefix, plus latest establishment event sequence number plus latest establishment event digest, plus indexed event signature. + These are endorsements by transferable AIDs that are not the controller + may be watchers or others. When latest establishment event is multisig then there will be multiple quadruples one per signing key, each a dup at same db key. dgKey @@ -601,37 +765,45 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus digest of serialized event More than one value per DB key is allowed - .kels is named sub DB of key event log tables that map sequence numbers - to serialized event digests. + .pses is named sub DB of partially signed key event escrows + that each map pre + sequence number to serialized event digest. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pses is named sub DB of partially signed escrowed event tables - that map sequence numbers to serialized event digests. + .pwes is named sub DB of partially witnessed key event escrowes + that each map pre + sequence number to serialized event digest. + these are for escrows of events with verified signatures but not + yet verified witness reciepts. snKey Values are digests used to lookup event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .pdes is named sub DB of partially delegated escrowed couples - that map digest to seal source couple that provides source - (delegator or issuer) event seal. Each couples is concatenations - of full qualified items, snu+dig of authorizing (delegating or - issuing) source event. - dgKey - Values are couples used to lookup source event in .kels sub DB - DB is keyed by identifier prefix plus digest of key event - Only one value per DB key is allowed - - .pwes is named sub DB of partially witnessed escrowed event tables - that map sequence numbers to serialized event digests. + .pdes is named sub DB of partially delegated key event escrows + that each map pre + sequence number to serialized event digest. This is + used in conjunction with .udes which escrows the associated seal + source couple. snKey - Values are digests used to lookup event in .evts sub DB + Values are digests used to lookup delegated event in .evts sub DB DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed + .udes is named sub DB of unverified delegation seal source couple escrows + that map (pre, digest) of delegated event to delegating seal source + couple (sn, dig) that provides source delegator event seal. + Each couple is concatenation of fully qualified items, snu+dig + of delegating source event in which seal of delegated event appears. + dgKey + Values are serialized instances of CatCesrSuber as couples + (Seqner.qb64b, Saider.qb64b) used to lookup source event in delegator's + KEL. + DB is keyed by identifier prefix plus digest of key event + Only one value per DB key is allowed + Once escrow is accepted then delegation approval source seal couples + go into .aess database of authorizing event source seal couples + .uwes is named sub DB of unverified event indexed escrowed couples from witness signers. Witnesses are from witness list of latest establishment event for the receipted event. Each couple is concatenation of fully @@ -669,12 +841,8 @@ class Baser(dbing.LMDBer): DB is keyed by identifier prefix plus sequence number of key event More than one value per DB key is allowed - .fons is named subDB instance of MatterSuber that maps - (prefix, digest) e.g. dgKey to fn value (first seen ordinal number) of - the associated event. So one can lookup event digest, get its fn here - and then use fn to fetch event by fn from .fels. - .states (stts) is named subDB instance of SerderSuber that maps a prefix + .states (subkey stts.) is named subDB instance of SerderSuber that maps a prefix to the latest keystate for that prefix. Used by ._kevers.db for read through cache of key state to reload kevers in memory @@ -757,6 +925,20 @@ class Baser(dbing.LMDBer): identical message bodies across participants in group multisig body trying to reach concensus on events or credentials. + .pubs is CatCesrIoSetSuber with subkey="pubs." of concatenated tuples + (qb64 pre, qb64 snh) indexed by qb64 of public key. Maps each signing + public key from establishment event to the events's prefix and sequence number + so can look up an event by any of its signing keys. Updated by Kever.logEvent + + .digs is CatCesrIoSetSuber with subkey="digs." of of concatenated tuples + (qb64 pre, qb64 snh) indexed by qb64 of digest of next signing public key. + Maps each next signing public key digest from establishment event to + the events's prefix and sequence number so can look up an event by any + of its next public signing key digests. Updated by Kever.logEvent + + Missing ToDo XXXX other attributes as sub dbs not documented here + such as .wits etc + Properties: kevers (dbdict): read through cache of kevers of states for KELs in db @@ -780,15 +962,18 @@ def __init__(self, headDirPath=None, reopen=False, **kwa): """ - self.prefixes = oset() + self.prefixes = oset() # should change to hids for hab ids + self.groups = oset() # group hab ids self._kevers = dbdict() self._kevers.db = self # assign db for read through cache of kevers - if (mapSize := os.getenv(KERIBaserMapSizeKey)) is not None: + mapSize = os.getenv(dbing.KERIBaserMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: try: self.MapSize = int(mapSize) except ValueError: - logger.error("KERI_BASER_MAP_SIZE must be an integer value >1!") + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIBaserMapSizeKey} or {dbing.KERILMDBMapSizeKey}") raise super(Baser, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) @@ -824,6 +1009,7 @@ def reopen(self, **kwa): self.evts = self.env.open_db(key=b'evts.') self.fels = self.env.open_db(key=b'fels.') + self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.dtss = self.env.open_db(key=b'dtss.') self.aess = self.env.open_db(key=b'aess.') self.sigs = self.env.open_db(key=b'sigs.', dupsort=True) @@ -832,23 +1018,36 @@ def reopen(self, **kwa): self.ures = self.env.open_db(key=b'ures.', dupsort=True) self.vrcs = self.env.open_db(key=b'vrcs.', dupsort=True) self.vres = self.env.open_db(key=b'vres.', dupsort=True) - self.kels = self.env.open_db(key=b'kels.', dupsort=True) self.pses = self.env.open_db(key=b'pses.', dupsort=True) - self.pdes = self.env.open_db(key=b'pdes.') self.pwes = self.env.open_db(key=b'pwes.', dupsort=True) + self.pdes = subing.OnIoDupSuber(db=self, subkey='pdes.') + self.udes = subing.CatCesrSuber(db=self, subkey='udes.', + klas=(coring.Seqner, coring.Saider)) self.uwes = self.env.open_db(key=b'uwes.', dupsort=True) self.ooes = self.env.open_db(key=b'ooes.', dupsort=True) self.dels = self.env.open_db(key=b'dels.', dupsort=True) self.ldes = self.env.open_db(key=b'ldes.', dupsort=True) - self.qnfs = self.env.open_db(key=b'qnfs.', dupsort=True) + self.qnfs = subing.IoSetSuber(db=self, subkey="qnfs.", dupsort=True) # events as ordered by first seen ordinals - self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=coring.Seqner) + self.fons = subing.CesrSuber(db=self, subkey='fons.', klas=core.Number) self.migs = subing.CesrSuber(db=self, subkey="migs.", klas=coring.Dater) self.vers = subing.Suber(db=self, subkey="vers.") + # event source local (protected) or non-local (remote not protected) + self.esrs = koming.Komer(db=self, + schema=EventSourceRecord, + subkey='esrs.') + + # misfit escrows whose processing may change the .esrs event source record + self.misfits = subing.IoSetSuber(db=self, subkey='mfes.') + + # delegable events escrows. events with local delegator that need approval + self.delegables = subing.IoSetSuber(db=self, subkey='dees.') + # Kever state made of KeyStateRecord key states + # TODO: clean self.states = koming.Komer(db=self, schema=KeyStateRecord, subkey='stts.') @@ -859,11 +1058,8 @@ def reopen(self, **kwa): self.habs = koming.Komer(db=self, subkey='habs.', schema=HabitatRecord, ) - - # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix - self.nmsp = koming.Komer(db=self, - subkey='nmsp.', - schema=HabitatRecord, ) + # habitat name database mapping (domain,name) as key to Prefixer + self.names = subing.Suber(db=self, subkey='names.', sep="^") # SAD support datetime stamps and signatures indexed and not-indexed # all sad sdts (sad datetime serializations) maps said to date-time @@ -873,7 +1069,7 @@ def reopen(self, **kwa): # given by quadruple (saider.qb64, prefixer.qb64, seqner.q64, diger.qb64) # of reply and trans signer's key state est evt to val Siger for each # signature. - self.ssgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=coring.Siger) + self.ssgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=indexing.Siger) # all sad scgs (sad non-indexed signature serializations) maps SAD SAID # to couple (Verfer, Cigar) of nontrans signer of signature in Cigar @@ -884,6 +1080,7 @@ def reopen(self, **kwa): # all reply messages. Maps reply said to serialization. Replys are # versioned sads ( with version string) so use Serder to deserialize and # use .sdts, .ssgs, and .scgs for datetimes and signatures + # TODO: clean self.rpys = subing.SerderSuber(db=self, subkey='rpys.') # all reply escrows indices of partially signed reply messages. Maps @@ -894,6 +1091,7 @@ def reopen(self, **kwa): # auth AuthN/AuthZ by controller at cid of endpoint provider at eid # maps key=cid.role.eid to val=said of end reply + # TODO: clean self.eans = subing.CesrSuber(db=self, subkey='eans.', klas=coring.Saider) # auth AuthN/AuthZ by endpoint provider at eid of location at scheme url @@ -910,76 +1108,102 @@ def reopen(self, **kwa): self.locs = koming.Komer(db=self, subkey='locs.', schema=LocationRecord, ) + # observed oids by watcher by cid.aid.oid (endpoint identifier) + # data extracted from reply loc + self.obvs = koming.Komer(db=self, + subkey='obvs.', + schema=ObservedRecord, ) # index of last retrieved message from witness mailbox + # TODO: clean self.tops = koming.Komer(db=self, subkey='witm.', schema=TopicsRecord, ) # group partial signature escrow self.gpse = subing.CatCesrIoSetSuber(db=self, subkey='gpse.', - klas=(coring.Seqner, coring.Saider)) + klas=(core.Number, coring.Saider)) # group delegate escrow self.gdee = subing.CatCesrIoSetSuber(db=self, subkey='gdee.', - klas=(coring.Seqner, coring.Saider)) + klas=(core.Number, coring.Saider)) # group partial witness escrow self.gpwe = subing.CatCesrIoSetSuber(db=self, subkey='gdwe.', - klas=(coring.Seqner, coring.Saider)) + klas=(core.Number, coring.Saider)) # completed group multisig + # TODO: clean self.cgms = subing.CesrSuber(db=self, subkey='cgms.', klas=coring.Saider) # exchange message partial signature escrow self.epse = subing.SerderSuber(db=self, subkey="epse.") + # exchange message PS escrow date time of message + self.epsd = subing.CesrSuber(db=self, subkey="epsd.", + klas=coring.Dater) + # exchange messages + # TODO: clean self.exns = subing.SerderSuber(db=self, subkey="exns.") # Forward pointer to a provided reply message + # TODO: clean self.erpy = subing.CesrSuber(db=self, subkey="erpy.", klas=coring.Saider) - # exchange messages - self.sxns = subing.SerderSuber(db=self, subkey="sxns.") - # exchange message signatures - self.esigs = subing.CesrIoSetSuber(db=self, subkey='esigs.', klas=coring.Siger) + # TODO: clean + self.esigs = subing.CesrIoSetSuber(db=self, subkey='esigs.', klas=indexing.Siger) # exchange message signatures + # TODO: clean self.ecigs = subing.CatCesrIoSetSuber(db=self, subkey='ecigs.', klas=(coring.Verfer, coring.Cigar)) # exchange pathed attachments + # TODO: clean self.epath = subing.IoSetSuber(db=self, subkey=".epath") + self.essrs = subing.CesrIoSetSuber(db=self, subkey=".essrs", klass=coring.Texter) + # accepted signed 12-word challenge response exn messages keys by prefix of signer + # TODO: clean self.chas = subing.CesrIoSetSuber(db=self, subkey='chas.', klas=coring.Saider) # successfull signed 12-word challenge response exn messages keys by prefix of signer + # TODO: clean self.reps = subing.CesrIoSetSuber(db=self, subkey='reps.', klas=coring.Saider) # authorzied well known OOBIs + # TODO: clean self.wkas = koming.IoSetKomer(db=self, subkey='wkas.', schema=WellKnownAuthN) # KSN support datetime stamps and signatures indexed and not-indexed # all ksn kdts (key state datetime serializations) maps said to date-time + # TODO: clean self.kdts = subing.CesrSuber(db=self, subkey='kdts.', klas=coring.Dater) # all key state messages. Maps key state said to serialization. ksns are # KeyStateRecords so use ._asdict or ._asjson as appropriate # use .kdts, .ksgs, and .kcgs for datetimes and signatures + # TODO: clean self.ksns = koming.Komer(db=self, schema=KeyStateRecord, subkey='ksns.') - #self.ksns = subing.SerderSuber(db=self, subkey='ksns.') # key state SAID database for successfully saved key state notices # maps key=(prefix, aid) to val=said of key state + # TODO: clean self.knas = subing.CesrSuber(db=self, subkey='knas.', klas=coring.Saider) + # Watcher watched SAID database for successfully saved watched AIDs for a watcher + # maps key=(cid, aid, oid) to val=said of rpy message + # TODO: clean + self.wwas = subing.CesrSuber(db=self, subkey='wwas.', klas=coring.Saider) + # config loaded oobis to be processed asynchronously, keyed by oobi URL + # TODO: clean self.oobis = koming.Komer(db=self, subkey='oobis.', schema=OobiRecord, @@ -998,40 +1222,47 @@ def reopen(self, **kwa): sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Resolved OOBIs (those that have been processed successfully for this database. + # TODO: clean self.roobi = koming.Komer(db=self, subkey='roobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Well known OOBIs that are to be used for mfa against a resolved OOBI. + # TODO: clean self.woobi = koming.Komer(db=self, subkey='woobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Well known OOBIs that are to be used for mfa against a resolved OOBI. + # TODO: clean self.moobi = koming.Komer(db=self, subkey='moobi.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Multifactor well known OOBI auth records to process. Keys by controller URL + # TODO: clean self.mfa = koming.Komer(db=self, subkey='mfa.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # Resolved multifactor well known OOBI auth records. Keys by controller URL + # TODO: clean self.rmfa = koming.Komer(db=self, - subkey='mfa.', + subkey='rmfa.', schema=OobiRecord, sep=">") # Use seperator not a allowed in URLs so no splitting occurs. # JSON schema SADs keys by the SAID + # TODO: clean self.schema = subing.SchemerSuber(db=self, subkey='schema.') # Field values for contact information for remote identifiers. Keyed by prefix/field + # TODO: clean self.cfld = subing.Suber(db=self, subkey="cfld.") @@ -1039,13 +1270,16 @@ def reopen(self, **kwa): self.hbys = subing.Suber(db=self, subkey='hbys.') # Signed contact data, keys by prefix + # TODO: clean self.cons = subing.Suber(db=self, subkey="cons.") # Transferable signatures on contact data + # TODO: clean self.ccigs = subing.CesrSuber(db=self, subkey='ccigs.', klas=coring.Cigar) # Chunked image data for contact information for remote identifiers + # TODO: clean self.imgs = self.env.open_db(key=b'imgs.') # Delegation escrow dbs # @@ -1055,22 +1289,20 @@ def reopen(self, **kwa): # delegated unanchored escrow self.dune = subing.SerderSuber(db=self, subkey='dune.') - # completed group multisig + # delegate publication escrow for sending delegator info to my witnesses + self.dpub = subing.SerderSuber(db=self, subkey='dpub.') + + # completed group delegated AIDs + # TODO: clean self.cdel = subing.CesrSuber(db=self, subkey='cdel.', klas=coring.Saider) - # public keys mapped to the AID and event seq no they appeared in - self.pubs = subing.CatCesrIoSetSuber(db=self, subkey="pubs.", - klas=(coring.Prefixer, coring.Seqner)) - - # next key digests mapped to the AID and event seq no they appeared in - self.digs = subing.CatCesrIoSetSuber(db=self, subkey="digs.", - klas=(coring.Prefixer, coring.Seqner)) - # multisig sig embed payload SAID mapped to containing exn messages across group multisig participants + # TODO: clean self.meids = subing.CesrIoSetSuber(db=self, subkey="meids.", klas=coring.Saider) # multisig sig embed payload SAID mapped to group multisig participants AIDs + # TODO: clean self.maids = subing.CesrIoSetSuber(db=self, subkey="maids.", klas=coring.Prefixer) self.reload() @@ -1092,38 +1324,20 @@ def reload(self): try: kever = eventing.Kever(state=ksr, db=self, - prefixes=self.prefixes, local=True) except kering.MissingEntryError as ex: # no kel event for keystate removes.append(keys) # remove from .habs continue self.kevers[kever.prefixer.qb64] = kever self.prefixes.add(kever.prefixer.qb64) - elif data.mid is None: # in .habs but no corresponding key state and not a group so remove - removes.append(keys) # no key state or KEL event for .hab record + if data.mid: # group hab + self.groups.add(data.hid) - for keys in removes: # remove bare .habs records - self.habs.rem(keys=keys) - - # Load namespaced Habs - removes = [] - for keys, data in self.nmsp.getItemIter(): - if (ksr := self.states.get(keys=data.hid)) is not None: - try: - kever = eventing.Kever(state=ksr, - db=self, - prefixes=self.prefixes, - local=True) - except kering.MissingEntryError as ex: # no kel event for keystate - removes.append(keys) # remove from .habs - continue - self.kevers[kever.prefixer.qb64] = kever - self.prefixes.add(kever.prefixer.qb64) elif data.mid is None: # in .habs but no corresponding key state and not a group so remove removes.append(keys) # no key state or KEL event for .hab record for keys in removes: # remove bare .habs records - self.nmsp.rem(keys=keys) + self.habs.rem(keys=keys) def migrate(self): """ Run all migrations required @@ -1140,13 +1354,14 @@ def migrate(self): ver = semver.VersionInfo.parse(keri.__version__) ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) if self.version is not None and semver.compare(version, str(ver_no_prerelease)) > 0: - print(f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") + print( + f"Skipping migration {version} as higher than the current KERI version {keri.__version__}") continue - # Check to see if migration version is for an older database version + # Skip migrations already run - where version less than (-1) or equal to (0) database version if self.version is not None and semver.compare(version, self.version) != 1: continue - print(f"Migrating database v{self.version} --> v{version} ...") + print(f"Migrating database v{self.version} --> v{version}") for migration in migrations: modName = f"keri.db.migrations.{migration}" if self.migs.get(keys=(migration,)) is not None: @@ -1154,6 +1369,7 @@ def migrate(self): mod = importlib.import_module(modName) try: + print(f"running migration {modName}") mod.migrate(self) except Exception as e: print(f"\nAbandoning migration {migration} at version {version} with error: {e}") @@ -1170,73 +1386,29 @@ def clearEscrows(self): """ Clear all escrows """ - count = 0 for (k, _) in self.getUreItemIter(): - count += 1 self.delUres(key=k) - logger.info(f"KEL: Cleared {count} unverified receipt escrows") - - count = 0 for (k, _) in self.getVreItemIter(): - count += 1 self.delVres(key=k) - logger.info(f"KEL: Cleared {count} verified receipt escrows") - - count = 0 - for (k, _) in self.getPseItemsNextIter(): - count += 1 + for (k, _) in self.getPseItemIter(): self.delPses(key=k) - logger.info(f"KEL: Cleared {count} partially signed escrows") - - count = 0 for (k, _) in self.getPweItemIter(): - count += 1 self.delPwes(key=k) - logger.info(f"KEL: Cleared {count} partially witnessed escrows") - - count = 0 for (k, _) in self.getUweItemIter(): - count += 1 self.delUwes(key=k) - logger.info(f"KEL: Cleared {count} unverified event indexed escrowed couples") - - count = 0 for (k, _) in self.getOoeItemIter(): - count += 1 self.delOoes(key=k) - logger.info(f"KEL: Cleared {count} out of order escrows") - - count = 0 for (k, _) in self.getLdeItemIter(): - count += 1 self.delLdes(key=k) - logger.info(f"KEL: Cleared {count} likely duplicitous escrows") - - count = 0 - for k, _ in self.getQnfItemsNextIter(): - self.delQnfs(key=k) - logger.info(f"KEL: Cleared {count} query not found escrows") - - count = 0 - for (key, _) in self.getPdeItemsNextIter(): - count += 1 - self.delPde(key=key) - logger.info(f"KEL: Cleared {count} partially delegated key event escrows") - - for name, escrow, desc in [ - ('rpes', self.rpes, 'reply escrows'), - ('eoobi', self.eoobi, 'failed, retryable OOBI escrow'), - ('gpwe', self.gpwe, 'group partial witness escrow'), - ('gdee', self.gdee, 'group delegate escrow'), - ('dpwe', self.dpwe, 'delegated partial witness escrow'), - ('gpse', self.gpse, 'group partial signature escrow'), - ('epse', self.epse, 'exchange partial signature escrow'), - ('dune', self.dune, 'delegated unanchored escrow')]: + for (pre, said), edig in self.qnfs.getItemIter(): + self.qnfs.rem(keys=(pre, said)) + + + for escrow in [self.qnfs, self.misfits, self.delegables, self.pdes, self.udes, self.rpes, self.epsd, self.eoobi, + self.dpub, self.gpwe, self.gdee, self.dpwe, self.gpse, self.epse, self.dune]: count = escrow.cntAll() escrow.trim() - logger.info(f"KEL: Cleared {count} escrows from ({name.ljust(5)}): {desc}") - - logger.info("Cleared KEL escrows") + logger.info(f"KEL: Cleared {count} escrows from ({escrow}") @property def current(self): @@ -1252,7 +1424,6 @@ def current(self): if self.version == keri.__version__: return True - # If database version is ahead of library version, throw exception ver = semver.VersionInfo.parse(keri.__version__) ver_no_prerelease = semver.Version(ver.major, ver.minor, ver.patch) if self.version is not None and semver.compare(self.version, str(ver_no_prerelease)) == 1: @@ -1281,7 +1452,7 @@ def complete(self, name=None): migrations = [] if not name: for version, migs in MIGRATIONS: - # Only get migration completion dates for migrations that have been run + # Print entries only for migrations that have been run if self.version is not None and semver.compare(version, self.version) <= 0: for mig in migs: dater = self.migs.get(keys=(mig,)) @@ -1308,7 +1479,7 @@ def clean(self): temp=self.temp, headDirPath=self.headDirPath, perm=self.perm, - clean=True) as copy: + clean=True) as copy: # copy is Baser instance with reopenDB(db=self, reuse=True, readonly=True): # reopen as readonly if not os.path.exists(self.path): @@ -1326,27 +1497,54 @@ def clean(self): for msg in self.cloneAllPreIter(): # clone into copy psr.parseOne(ims=msg) + # This is the list of non-set based databases that are not created as part of event processing. + # for now we are just copying them from self to copy without worrying about being able to + # reprocess them. We need a more secure method in the future + unsecured = ["hbys", "schema", "states", "rpys", "eans", "tops", "cgms", "exns", "erpy", + "kdts", "ksns", "knas", "oobis", "roobi", "woobi", "moobi", "mfa", "rmfa", + "cfld", "cons", "ccigs", "cdel", "migs"] + + for name in unsecured: + srcdb = getattr(self, name) + cpydb = getattr(copy, name) + for keys, val in srcdb.getItemIter(): + cpydb.put(keys=keys, val=val) + + # This is the list of set based databases that are not created as part of event processing. + # for now we are just copying them from self to copy without worrying about being able to + # reprocess them. We need a more secure method in the future + sets = ["esigs", "ecigs", "epath", "chas", "reps", "wkas", "meids", "maids"] + for name in sets: + srcdb = getattr(self, name) + cpydb = getattr(copy, name) + for keys, val in srcdb.getItemIter(): + cpydb.add(keys=keys, val=val) + + # Insecure raw imgs database copy. + for (key, val) in self.getTopItemIter(self.imgs): + copy.imgs.setVal(key=key, val=val) + # clone .habs habitat name prefix Komer subdb # copy.habs = koming.Komer(db=copy, schema=HabitatRecord, subkey='habs.') # copy for keys, val in self.habs.getItemIter(): if val.hid in copy.kevers: # only copy habs that verified copy.habs.put(keys=keys, val=val) + ns = "" if val.domain is None else val.domain + copy.names.put(keys=(ns, val.name), val=val.hid) copy.prefixes.add(val.hid) - - if not copy.habs.get(keys=(self.name,)): - raise ValueError("Error cloning habs, missing orig name={}." - "".format(self.name)) + if val.mid: # a group hab + copy.groups.add(val.hid) # clone .ends and .locs databases - for keys, val in self.ends.getItemIter(): + for (cid, role, eid), val in self.ends.getItemIter(): exists = False # only copy if entries in both .ends and .locs for scheme in ("https", "http", "tcp"): # all supported schemes - lval = self.locs.get(keys=(val.eid, scheme)) - if lval and lval.cid == keys[0] and lval.role == keys[1]: + lval = self.locs.get(keys=(eid, scheme)) + if lval: exists = True # loc with matching cid and rol - copy.locs.put(keys=(val.eid, scheme), val=lval) + copy.locs.put(keys=(eid, scheme), val=lval) if exists: # only copy end if has at least one matching loc - copy.ends.put(keys=keys, vals=[val]) + copy.ends.put(keys=(cid, role, eid), val=val) # remove own db directory replace with clean clone copy if os.path.exists(self.path): @@ -1369,6 +1567,10 @@ def clean(self): self.prefixes.clear() self.prefixes.update(copy.prefixes) + # clear and clone .gids + self.groups.clear() + self.groups.update(copy.groups) + with reopenDB(db=self, reuse=True): # make sure can reopen if not isinstance(self.env, lmdb.Environment): raise ValueError("Error cloning, unable to reopen." @@ -1378,17 +1580,23 @@ def clean(self): if os.path.exists(copy.path): shutil.rmtree(copy.path) - def clonePreIter(self, pre, fn=0): """ Returns iterator of first seen event messages with attachments for the identifier prefix pre starting at first seen order number, fn. Essentially a replay in first seen order with attachments + + Parameters: + pre is bytes of itdentifier prefix + fn is int fn to resume replay. Earliset is fn=0 + + Returns: + msgs (Iterator): over all items with pre starting at fn """ if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getFelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getFelItemPreIter(pre, fn=fn): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1396,18 +1604,19 @@ def clonePreIter(self, pre, fn=0): yield msg - def cloneAllPreIter(self, key=b''): + def cloneAllPreIter(self): """ Returns iterator of first seen event messages with attachments for all - identifier prefixes starting at key. If key == b'' then rstart at first + identifier prefixes starting at key. If key == b'' then start at first key in databse. Use key to resume replay. Essentially a replay in first seen order with attachments of entire set of FELs. - Parameters: - key (bytes): fnKey(pre, fn) + Returns: + msgs (Iterator): over all items in db + """ - for pre, fn, dig in self.getFelItemAllPreIter(key=key): + for pre, fn, dig in self.getFelItemAllPreIter(): try: msg = self.cloneEvtMsg(pre=pre, fn=fn, dig=dig) except Exception: @@ -1437,61 +1646,59 @@ def cloneEvtMsg(self, pre, fn, dig): # add indexed signatures to attachments if not (sigs := self.getSigs(key=dgkey)): raise kering.MissingEntryError("Missing sigs for dig={}.".format(dig)) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigs)).qb64b) + atc.extend(core.Counter(code=core.Codens.ControllerIdxSigs, + count=len(sigs), gvrsn=kering.Vrsn_1_0).qb64b) for sig in sigs: atc.extend(sig) # add indexed witness signatures to attachments if wigs := self.getWigs(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.WitnessIdxSigs, - count=len(wigs)).qb64b) + atc.extend(core.Counter(code=core.Codens.WitnessIdxSigs, + count=len(wigs), gvrsn=kering.Vrsn_1_0).qb64b) for wig in wigs: atc.extend(wig) # add authorizer (delegator/issuer) source seal event couple to attachments couple = self.getAes(dgkey) if couple is not None: - atc.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + atc.extend(core.Counter(code=core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(couple) - elif self.kevers[pre].delegated: - if serdering.SerderKERI(raw=bytes(raw)).estive: - raise kering.MissingEntryError("Missing delegator anchor seal for dig={}.".format(dig)) - # add trans receipts quadruples to attachments + # add trans endorsement quadruples to attachments not controller + # may have been originally key event attachments or receipted endorsements if quads := self.getVrcs(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.TransReceiptQuadruples, - count=len(quads)).qb64b) + atc.extend(core.Counter(code=core.Codens.TransReceiptQuadruples, + count=len(quads), gvrsn=kering.Vrsn_1_0).qb64b) for quad in quads: atc.extend(quad) - # add nontrans receipts couples to attachments + # add nontrans endorsement couples to attachments not witnesses + # may have been originally key event attachments or receipted endorsements if coups := self.getRcts(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, - count=len(coups)).qb64b) + atc.extend(core.Counter(code=core.Codens.NonTransReceiptCouples, + count=len(coups), gvrsn=kering.Vrsn_1_0).qb64b) for coup in coups: atc.extend(coup) # add first seen replay couple to attachments if not (dts := self.getDts(key=dgkey)): raise kering.MissingEntryError("Missing datetime for dig={}.".format(dig)) - atc.extend(coring.Counter(code=coring.CtrDex.FirstSeenReplayCouples, - count=1).qb64b) - atc.extend(coring.Seqner(sn=fn).qb64b) + atc.extend(core.Counter(code=core.Codens.FirstSeenReplayCouples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) + atc.extend(core.Number(num=fn, code=core.NumDex.Huge).qb64b) # may not need to be Huge atc.extend(coring.Dater(dts=bytes(dts)).qb64b) # prepend pipelining counter to attachments if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - pcnt = coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, - count=(len(atc) // 4)).qb64b + pcnt = core.Counter(code=core.Codens.AttachmentGroup, + count=(len(atc) // 4), gvrsn=kering.Vrsn_1_0).qb64b msg.extend(pcnt) msg.extend(atc) return msg - def cloneDelegation(self, kever): """ Recursively clone delegation chain from AID of Kever if one exits. @@ -1500,15 +1707,14 @@ def cloneDelegation(self, kever): kever (Kever): Kever from which to clone the delegator's AID. """ - if kever.delegated: - dkever = self.kevers[kever.delegator] + if kever.delegated and kever.delpre in self.kevers: + dkever = self.kevers[kever.delpre] yield from self.cloneDelegation(dkever) - for dmsg in self.clonePreIter(pre=kever.delegator, fn=0): + for dmsg in self.clonePreIter(pre=kever.delpre, fn=0): yield dmsg - - def findAnchoringSealEvent(self, pre, seal, sn=0): + def fetchAllSealingEventByEventSeal(self, pre, seal, sn=0): """ Search through a KEL for the event that contains a specific anchored SealEvent type of provided seal but in dict form and is also fully @@ -1525,118 +1731,88 @@ def findAnchoringSealEvent(self, pre, seal, sn=0): sn (int): beginning sn to search """ - if tuple(seal.keys()) != eventing.SealEvent._fields: # wrong type of seal + if tuple(seal) != eventing.SealEvent._fields: # wrong type of seal return None seal = eventing.SealEvent(**seal) #convert to namedtuple for evt in self.getEvtPreIter(pre=pre, sn=sn): # includes disputed & superseded srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: - if tuple(eseal.keys()) == eventing.SealEvent._fields: + for eseal in srdr.seals or []: # or [] for seals 'a' field missing + if tuple(eseal) == eventing.SealEvent._fields: eseal = eventing.SealEvent(**eseal) # convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): return srdr return None + # use alias here until can change everywhere for backwards compatibility + findAnchoringSealEvent = fetchAllSealingEventByEventSeal # alias - def findAnchoringSeal(self, pre, seal, sn=0): + def fetchLastSealingEventByEventSeal(self, pre, seal, sn=0): """ - Search through a KEL for the event that contains an anchored - Seal with same Seal type as provided seal but in dict form. - Searchs from sn forward (default = 0). Only searches last event at any - sn therefore does not search any disputed or superseded events. - Returns the Serder of the first event with the anchored Seal seal, - None if not found + Search through a KEL for the last event at any sn but that contains a + specific anchored event seal of namedtuple SealEvent type that matches + the provided seal in dict form and is also fully witnessed. + Searchs from provided sn forward (default = 0). + Searches only last events in KEL of pre so does not include disputed + and/or superseded events. + + Returns: + srdr (Serder): instance of the first event with the matching + anchoring SealEvent seal, + None if not found Parameters: pre (bytes|str): identifier of the KEL to search - seal (dict): dict form of Seal of any type to find in anchored + seal (dict): dict form of Seal of any type SealEvent to find in anchored seals list of each event sn (int): beginning sn to search """ - # create generic Seal namedtuple class using keys from provided seal dict - Seal = namedtuple('Seal', seal.keys()) # matching type - - for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # only last evt at sn - srdr = serdering.SerderKERI(raw=evt.tobytes()) - for eseal in srdr.seals or []: - if tuple(eseal.keys()) == Seal._fields: # same type of seal - eseal = Seal(**eseal) #convert to namedtuple - if seal == eseal and self.fullyWitnessed(srdr): - return srdr - return None - - - - def findAnchoringSealEventClone(self, pre, seal): - """ - Search through a KEL for the event that contains a specific anchored - SealEvent type of provided seal but in dict form. - Returns the Serder of the first event with the anchored SealEvent seal, - None if not found - Searchs from inception forward - - Parameters: - pre is qb64 identifier of the KEL to search - seal is dict form of SealEvent to find in anchored seals list of each event - - """ - if tuple(seal.keys()) != eventing.SealEvent._fields: # wrong type of seal + if tuple(seal) != eventing.SealEvent._fields: # wrong type of seal return None - #raise ValueError(f"Expected SealEvent got {seal}.") seal = eventing.SealEvent(**seal) #convert to namedtuple - # getEvtPreIter getEvtLastPreIter - - for evt in self.clonePreIter(pre=pre): # all events including superseded - srdr = serdering.SerderKERI(raw=evt) - for eseal in srdr.seals or []: - if tuple(eseal.keys()) == eventing.SealEvent._fields: - eseal = eventing.SealEvent(**eseal) #convert to namedtuple + for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # no disputed or superseded + srdr = serdering.SerderKERI(raw=evt.tobytes()) + for eseal in srdr.seals or []: # or [] for seals 'a' field missing + if tuple(eseal) == eventing.SealEvent._fields: + eseal = eventing.SealEvent(**eseal) # convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): return srdr - #spre = anc["i"] - #ssn = int(anc["s"], 16) - #sdig = anc["d"] - - #if spre == seal["i"] and ssn == int(seal["s"], 16) \ - #and seal["d"] == sdig and self.fullyWitnessed(srdr): - #return srdr - return None - def findAnchoringSealClone(self, pre, seal): - """ - Search through a KEL for the event that contains an anchored - Seal with same Seal type as provided seal but in dict form. - Returns the Serder of the first event with the anchored Seal seal, + + def fetchLastSealingEventBySeal(self, pre, seal, sn=0): + """Only searches last event at any sn therefore does not search + any disputed or superseded events. + Search through last event at each sn in KEL for the event that contains + an anchored Seal with same Seal type as provided seal but in dict form. + Searchs from sn forward (default = 0). + Returns the Serder of the first found event with the anchored Seal seal, None if not found - Searchs from inception forward Parameters: - pre is qb64 identifier of the KEL to search - seal is dict form of Seal of any type to find in anchored seals list of each event + pre (bytes|str): identifier of the KEL to search + seal (dict): dict form of Seal of any type to find in anchored + seals list of each event + sn (int): beginning sn to search """ # create generic Seal namedtuple class using keys from provided seal dict - Seal = namedtuple('Seal', seal.keys()) # matching type - - # getEvtPreIter getEvtLastPreIter + Seal = namedtuple('Seal', list(seal)) # matching type - for evt in self.clonePreIter(pre=pre): # all events including superseded - srdr = serdering.SerderKERI(raw=evt) - for eseal in srdr.seals or []: - if tuple(eseal.keys()) == Seal._fields: # same type of seal + for evt in self.getEvtLastPreIter(pre=pre, sn=sn): # only last evt at sn + srdr = serdering.SerderKERI(raw=evt.tobytes()) + for eseal in srdr.seals or []: # or [] for seals 'a' field missing + if tuple(eseal) == Seal._fields: # same type of seal eseal = Seal(**eseal) #convert to namedtuple if seal == eseal and self.fullyWitnessed(srdr): return srdr return None - def signingMembers(self, pre: str): """ Find signing members of a multisig group aid. @@ -1649,22 +1825,10 @@ def signingMembers(self, pre: str): list: qb64 identifier prefixes of signing members for provided aid """ - members = [] - if pre not in self.kevers: - return members - - kever = self.kevers[pre] - for verfer in kever.verfers: - if (couples := self.pubs.get(keys=(verfer.qb64,))) is None: - continue - - for couple in couples: - prefixer, seqner = couple - if prefixer.qb64 != pre: # Rule out aid being queried - members.append(prefixer.qb64) - - return members + if (habord := self.habs.get(keys=(pre,))) is None: + return None + return habord.smids def rotationMembers(self, pre: str): """ Find rotation members of a multisig group aid. @@ -1677,21 +1841,10 @@ def rotationMembers(self, pre: str): Returns: list: qb64 identifier prefixes of rotation members for provided aid """ - members = [] - if pre not in self.kevers: - return members - - kever = self.kevers[pre] - for diger in kever.ndigers: - if (couples := self.digs.get(keys=(diger.qb64,))) is None: - continue - - for couple in couples: - prefixer, seqner = couple - if prefixer.qb64 != pre: # Rule out aid being queried - members.append(prefixer.qb64) + if (habord := self.habs.get(keys=(pre,))) is None: + return None - return members + return habord.rmids def fullyWitnessed(self, serder): """ Verify the witness threshold on the event @@ -1806,7 +1959,6 @@ def getEvtPreIter(self, pre, sn=0): for dig in self.getKelIter(pre, sn=sn): try: - dgkey = dbing.dgKey(pre, dig) # get message if not (raw := self.getEvt(key=dgkey)): raise kering.MissingEntryError("Missing event for dig={}.".format(dig)) @@ -1892,13 +2044,13 @@ def appendFe(self, pre, val): pre is bytes identifier prefix for event val is event digest """ - return self.appendOrdValPre(db=self.fels, pre=pre, val=val) + return self.appendOnVal(db=self.fels, key=pre, val=val) def getFelItemPreIter(self, pre, fn=0): """ - Returns iterator of all (fn, dig) duples in first seen order for all events - with same prefix, pre, in database. Items are sorted by fnKey(pre, fn) - where fn is first seen order number int. + Returns iterator of all (pre, fn, dig) triples in first seen order for + all events with same prefix, pre, in database. Items are sorted by + fnKey(pre, fn) where fn is first seen order number int. Returns a First Seen Event Log FEL. Returned items are duples of (fn, dig): Where fn is first seen order number int and dig is event digest for lookup in .evts sub db. @@ -1908,11 +2060,14 @@ def getFelItemPreIter(self, pre, fn=0): Parameters: pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 + + Returns: + items (Iterator[(pre, fn, val)]): over all items starting at pre, on """ - return self.getAllOrdItemPreIter(db=self.fels, pre=pre, on=fn) + return self.getOnItemIter(db=self.fels, key=pre, on=fn) - def getFelItemAllPreIter(self, key=b''): + def getFelItemAllPreIter(self): """ Returns iterator of all (pre, fn, dig) triples in first seen order for all events for all prefixes in database. Items are sorted by @@ -1928,7 +2083,8 @@ def getFelItemAllPreIter(self, key=b''): key is key location in db to resume replay, If empty then start at first key in database """ - return self.getAllOrdItemAllPreIter(db=self.fels, key=key) + #return self.getAllOnItemAllPreIter(db=self.fels, key=key) + return self.getOnItemIter(db=self.fels, key=b'') def putDts(self, key, val): """ @@ -2179,7 +2335,7 @@ def putUres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ures, key, vals) + return self.putIoDupVals(self.ures, key, vals) def addUre(self, key, val): """ @@ -2190,7 +2346,7 @@ def addUre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ures, key, val) + return self.addIoDupVal(self.ures, key, val) def getUres(self, key): """ @@ -2200,7 +2356,7 @@ def getUres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ures, key) + return self.getIoDupVals(self.ures, key) def getUresIter(self, key): """ @@ -2210,7 +2366,7 @@ def getUresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.ures, key) + return self.getIoDupValsIter(self.ures, key) def getUreLast(self, key): """ @@ -2220,39 +2376,12 @@ def getUreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ures, key) + return self.getIoDupValLast(self.ures, key) - def getUreItemIter(self, key=b''): - """ - Use sgKey() - Return iterator of partial signed escrowed event triple items at next - key after key. - Items is (key, val) where proem has already been stripped from val - val is triple dig+pre+cig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getTopIoDupItemIter(self.ures, key) - def getUreItemsNext(self, key=b'', skip=True): + def getUreItemIter(self, key=b''): """ Use snKey() - Return all dups of partial signed escrowed event triple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is triple dig+pre+cig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.ures, key, skip) - - def getUreItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() Return iterator of partial signed escrowed event triple items at next key after key. Items is (key, val) where proem has already been stripped from val @@ -2262,7 +2391,8 @@ def getUreItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.ures, key, skip) + return self.getTopIoDupItemIter(self.ures, key) + #return self.getIoDupItemsNextIter(self.ures, key, skip) def cntUres(self, key): """ @@ -2270,7 +2400,7 @@ def cntUres(self, key): Return count of receipt triplets at key Returns zero if no entry at key """ - return self.cntIoVals(self.ures, key) + return self.cntIoDupVals(self.ures, key) def delUres(self, key): """ @@ -2278,7 +2408,7 @@ def delUres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.ures, key) + return self.delIoDupVals(self.ures, key) def delUre(self, key, val): """ @@ -2290,7 +2420,7 @@ def delUre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ures, key, val) + return self.delIoDupVal(self.ures, key, val) def putVrcs(self, key, vals): """ @@ -2360,7 +2490,7 @@ def putVres(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.vres, key, vals) + return self.putIoDupVals(self.vres, key, vals) def addVre(self, key, val): """ @@ -2371,7 +2501,7 @@ def addVre(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.vres, key, val) + return self.addIoDupVal(self.vres, key, val) def getVres(self, key): """ @@ -2381,7 +2511,7 @@ def getVres(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.vres, key) + return self.getIoDupVals(self.vres, key) def getVresIter(self, key): """ @@ -2391,7 +2521,7 @@ def getVresIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.vres, key) + return self.getIoDupValsIter(self.vres, key) def getVreLast(self, key): """ @@ -2401,7 +2531,7 @@ def getVreLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.vres, key) + return self.getIoDupValLast(self.vres, key) def getVreItemIter(self, key=b''): """ @@ -2416,34 +2546,7 @@ def getVreItemIter(self, key=b''): Duplicates are retrieved in insertion order. """ return self.getTopIoDupItemIter(self.vres, key) - - def getVreItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event quintuple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is Quinlet is edig + spre + ssnu + sdig +sig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.vres, key, skip) - - def getVreItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of partial signed escrowed event quintuple items at next - key after key. - Items is (key, val) where proem has already been stripped from val - val is Quinlet is edig + spre + ssnu + sdig +sig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.vres, key, skip) + #return self.getIoDupItemsNextIter(self.vres, key, skip) def cntVres(self, key): """ @@ -2451,7 +2554,7 @@ def cntVres(self, key): Return count of receipt quinlets at key Returns zero if no entry at key """ - return self.cntIoVals(self.vres, key) + return self.cntIoDupVals(self.vres, key) def delVres(self, key): """ @@ -2459,7 +2562,7 @@ def delVres(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.vres, key) + return self.delIoDupVals(self.vres, key) def delVre(self, key, val): """ @@ -2471,7 +2574,7 @@ def delVre(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.vres, key, val) + return self.delIoDupVal(self.vres, key, val) def putKes(self, key, vals): """ @@ -2481,7 +2584,7 @@ def putKes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.kels, key, vals) + return self.putIoDupVals(self.kels, key, vals) def addKe(self, key, val): """ @@ -2491,7 +2594,7 @@ def addKe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.kels, key, val) + return self.addIoDupVal(self.kels, key, val) def getKes(self, key): """ @@ -2500,7 +2603,7 @@ def getKes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.kels, key) + return self.getIoDupVals(self.kels, key) def getKeLast(self, key): """ @@ -2509,7 +2612,7 @@ def getKeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.kels, key) + return self.getIoDupValLast(self.kels, key) def cntKes(self, key): """ @@ -2517,7 +2620,7 @@ def cntKes(self, key): Return count of dup key event dig val at key Returns zero if no entry at key """ - return self.cntIoVals(self.kels, key) + return self.cntIoDupVals(self.kels, key) def delKes(self, key): """ @@ -2525,7 +2628,7 @@ def delKes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.kels, key) + return self.delIoDupVals(self.kels, key) def getKelIter(self, pre, sn=0): @@ -2549,7 +2652,10 @@ def getKelIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreIter(self.kels, pre, on=sn) + + return (self.getOnIoDupValIter(self.kels, pre, on=sn)) + + #return self.getOnIoDupValsAllPreIter(self.kels, pre, on=sn) def getKelBackIter(self, pre, sn=0): @@ -2573,7 +2679,7 @@ def getKelBackIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAllPreBackIter(self.kels, pre, sn) + return self.getOnIoDupValBackIter(self.kels, pre, sn) def getKelLastIter(self, pre, sn=0): @@ -2597,7 +2703,7 @@ def getKelLastIter(self, pre, sn=0): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValLastAllPreIter(self.kels, pre, on=sn) + return self.getOnIoDupLastValIter(self.kels, pre, on=sn) def putPses(self, key, vals): @@ -2608,7 +2714,7 @@ def putPses(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pses, key, vals) + return self.putIoDupVals(self.pses, key, vals) def addPse(self, key, val): """ @@ -2618,7 +2724,7 @@ def addPse(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pses, key, val) + return self.addIoDupVal(self.pses, key, val) def getPses(self, key): """ @@ -2627,7 +2733,7 @@ def getPses(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pses, key) + return self.getIoDupVals(self.pses, key) def getPsesIter(self, key): """ @@ -2636,7 +2742,7 @@ def getPsesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pses, key) + return self.getIoDupValsIter(self.pses, key) def getPseLast(self, key): """ @@ -2645,21 +2751,9 @@ def getPseLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pses, key) + return self.getIoDupValLast(self.pses, key) - def getPseItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.pses, key, skip) - - def getPseItemsNextIter(self, key=b'', skip=True): + def getPseItemIter(self, key=b''): """ Use sgKey() Return iterator of partial signed escrowed event dig items at next key after key. @@ -2669,7 +2763,8 @@ def getPseItemsNextIter(self, key=b'', skip=True): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoItemsNextIter(self.pses, key, skip) + return self.getTopIoDupItemIter(self.pses, key) + #return self.getIoDupItemsNextIter(self.pses, key, skip) def cntPses(self, key): """ @@ -2677,7 +2772,7 @@ def cntPses(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pses, key) + return self.cntIoDupVals(self.pses, key) def delPses(self, key): """ @@ -2685,7 +2780,7 @@ def delPses(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pses, key) + return self.delIoDupVals(self.pses, key) def delPse(self, key, val): """ @@ -2697,60 +2792,8 @@ def delPse(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pses, key, val) + return self.delIoDupVal(self.pses, key, val) - def putPde(self, key, val): - """ - Use dgKey() - Write serialized event source couple to key (snu+dig) - Does not overwrite existing val if any - Returns True If val successfully written Else False - Returns False if key already exists - """ - return self.putVal(self.pdes, key, val) - - def setPde(self, key, val): - """ - Use dgKey() - Write serialized seal source couple to key (snu+dig) - Overwrites existing val if any - Returns True If val successfully written Else False - """ - return self.setVal(self.pdes, key, val) - - def getPde(self, key): - """ - Use dgKey() - Return seal source couple at key - Returns None if no entry at key - """ - return self.getVal(self.pdes, key) - - def getPdes(self, key): - """ - Use dgKey() - Return list of out of order escrow event dig vals at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoVals(self.pdes, key) - - def getPdeItemsNextIter(self, key=b'', skip=True): - """ - Use dgKey() - Return list of witnessed signed escrowed event dig vals at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.pdes, key, skip) - - def delPde(self, key): - """ - Use dgKey() - Deletes value at key. - Returns True If key exists in database Else False - """ - return self.delVal(self.pdes, key) def putPwes(self, key, vals): """ @@ -2760,7 +2803,7 @@ def putPwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.pwes, key, vals) + return self.putIoDupVals(self.pwes, key, vals) def addPwe(self, key, val): """ @@ -2770,7 +2813,7 @@ def addPwe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.pwes, key, val) + return self.addIoDupVal(self.pwes, key, val) def getPwes(self, key): """ @@ -2779,7 +2822,7 @@ def getPwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.pwes, key) + return self.getIoDupVals(self.pwes, key) def getPwesIter(self, key): """ @@ -2788,7 +2831,7 @@ def getPwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.pwes, key) + return self.getIoDupValsIter(self.pwes, key) def getPweLast(self, key): """ @@ -2797,7 +2840,7 @@ def getPweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.pwes, key) + return self.getIoDupValLast(self.pwes, key) def getPweItemIter(self, key=b''): """ @@ -2810,30 +2853,19 @@ def getPweItemIter(self, key=b''): Duplicates are retrieved in insertion order. """ return self.getTopIoDupItemIter(self.pwes, key) - - def getPweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial witnessed escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.pwes, key, skip) - - def getPweItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of partial witnessed escrowed event dig items at next key after key. - Items is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.pwes, key, skip) + #return self.getIoDupItemsNextIter(self.pwes, key, skip) + + #def getPweIoDupItemIter(self, key=b''): + #""" + #Use sgKey() + #Return iterator of partial witnessed escrowed event dig items at next key after key. + #Items is (key, val) where proem has already been stripped from val + #If key is b'' empty then returns dup items at first key. + #If skip is False and key is not b'' empty then returns dup items at key + #Raises StopIteration Error when empty + #Duplicates are retrieved in insertion order. + #""" + #return self.getTopIoDupItemIter(self.pwes, key) def cntPwes(self, key): """ @@ -2841,7 +2873,7 @@ def cntPwes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.pwes, key) + return self.cntIoDupVals(self.pwes, key) def delPwes(self, key): """ @@ -2849,7 +2881,7 @@ def delPwes(self, key): Deletes all values at key in db. Returns True If key exists in db Else False """ - return self.delIoVals(self.pwes, key) + return self.delIoDupVals(self.pwes, key) def delPwe(self, key, val): """ @@ -2861,7 +2893,7 @@ def delPwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.pwes, key, val) + return self.delIoDupVal(self.pwes, key, val) def putUwes(self, key, vals): """ @@ -2872,7 +2904,7 @@ def putUwes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.uwes, key, vals) + return self.putIoDupVals(self.uwes, key, vals) def addUwe(self, key, val): """ @@ -2883,7 +2915,7 @@ def addUwe(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.uwes, key, val) + return self.addIoDupVal(self.uwes, key, val) def getUwes(self, key): """ @@ -2893,7 +2925,7 @@ def getUwes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.uwes, key) + return self.getIoDupVals(self.uwes, key) def getUwesIter(self, key): """ @@ -2903,7 +2935,7 @@ def getUwesIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.uwes, key) + return self.getIoDupValsIter(self.uwes, key) def getUweLast(self, key): """ @@ -2913,7 +2945,7 @@ def getUweLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.uwes, key) + return self.getIoDupValLast(self.uwes, key) def getUweItemIter(self, key=b''): """ @@ -2928,34 +2960,7 @@ def getUweItemIter(self, key=b''): Duplicates are retrieved in insertion order. """ return self.getTopIoDupItemIter(self.uwes, key) - - def getUweItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of partial signed escrowed receipt couple items at next - key after key. - Item is (key, val) where proem has already been stripped from val - val is couple edig+wig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.uwes, key, skip) - - def getUweItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of partial signed escrowed receipt couple items at next - key after key. - Items is (key, val) where proem has already been stripped from val - val is couple edig+wig - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.uwes, key, skip) + #return self.getIoDupItemsNextIter(self.uwes, key, skip) def cntUwes(self, key): """ @@ -2963,7 +2968,7 @@ def cntUwes(self, key): Return count of receipt couples at key Returns zero if no entry at key """ - return self.cntIoVals(self.uwes, key) + return self.cntIoDupVals(self.uwes, key) def delUwes(self, key): """ @@ -2971,7 +2976,7 @@ def delUwes(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.uwes, key) + return self.delIoDupVals(self.uwes, key) def delUwe(self, key, val): """ @@ -2983,7 +2988,7 @@ def delUwe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.uwes, key, val) + return self.delIoDupVal(self.uwes, key, val) def putOoes(self, key, vals): """ @@ -2993,7 +2998,7 @@ def putOoes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ooes, key, vals) + return self.putIoDupVals(self.ooes, key, vals) def addOoe(self, key, val): """ @@ -3003,7 +3008,7 @@ def addOoe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ooes, key, val) + return self.addIoDupVal(self.ooes, key, val) def getOoes(self, key): """ @@ -3012,7 +3017,7 @@ def getOoes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ooes, key) + return self.getIoDupVals(self.ooes, key) def getOoeLast(self, key): """ @@ -3021,7 +3026,7 @@ def getOoeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ooes, key) + return self.getIoDupValLast(self.ooes, key) def getOoeItemIter(self, key=b''): """ @@ -3034,30 +3039,7 @@ def getOoeItemIter(self, key=b''): Duplicates are retrieved in insertion order. """ return self.getTopIoDupItemIter(self.ooes, key) - - def getOoeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.ooes, key, skip) - - def getOoeItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of out of order escrowed event dig items at next key after key. - Items is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.ooes, key, skip) + #return self.getIoDupItemsNextIter(self.ooes, key, skip) def cntOoes(self, key): """ @@ -3065,7 +3047,7 @@ def cntOoes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ooes, key) + return self.cntIoDupVals(self.ooes, key) def delOoes(self, key): """ @@ -3073,7 +3055,7 @@ def delOoes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ooes, key) + return self.delIoDupVals(self.ooes, key) def delOoe(self, key, val): """ @@ -3086,98 +3068,8 @@ def delOoe(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ooes, key, val) - - def putQnfs(self, key, vals): - """ - Use snKey() - Write each out of order escrow event dig entry from list of bytes vals to key - Adds to existing event indexes at key if any - Returns True If at least one of vals is added as dup, False otherwise - Duplicates are inserted in insertion order. - """ - return self.putIoVals(self.qnfs, key, vals) - - def addQnf(self, key, val): - """ - Use snKey() - Add out of order escrow val bytes as dup to key in db - Adds to existing event indexes at key if any - Returns True if written else False if dup val already exists - Duplicates are inserted in insertion order. - """ - return self.addIoVal(self.qnfs, key, val) - - def getQnfs(self, key): - """ - Use snKey() - Return list of out of order escrow event dig vals at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoVals(self.qnfs, key) - - def getQnfLast(self, key): - """ - Use snKey() - Return last inserted dup val of out of order escrow event dig vals at key - Returns None if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoValLast(self.qnfs, key) - - def getQnfItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of out of order escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.qnfs, key, skip) - - def getQnfItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of out of order escrowed event dig items at next key after key. - Items is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.qnfs, key, skip) - - def cntQnfs(self, key): - """ - Use snKey() - Return count of dup event dig at key - Returns zero if no entry at key - """ - return self.cntIoVals(self.qnfs, key) - - def delQnfs(self, key): - """ - Use snKey() - Deletes all values at key. - Returns True If key exists in database Else False - """ - return self.delIoVals(self.qnfs, key) - - def delQnf(self, key, val): - """ - Use snKey() - Deletes dup val at key in db. - Returns True If dup at exists in db Else False + return self.delIoDupVal(self.ooes, key, val) - Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace - val is dup val (does not include insertion ordering proem) - """ - return self.delIoVal(self.qnfs, key, val) def putDes(self, key, vals): """ @@ -3187,7 +3079,7 @@ def putDes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.dels, key, vals) + return self.putIoDupVals(self.dels, key, vals) def addDe(self, key, val): """ @@ -3197,7 +3089,7 @@ def addDe(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.dels, key, val) + return self.addIoDupVal(self.dels, key, val) def getDes(self, key): """ @@ -3206,7 +3098,7 @@ def getDes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.dels, key) + return self.getIoDupVals(self.dels, key) def getDeLast(self, key): """ @@ -3216,7 +3108,7 @@ def getDeLast(self, key): Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.dels, key) + return self.getIoDupValLast(self.dels, key) def cntDes(self, key): """ @@ -3224,7 +3116,7 @@ def cntDes(self, key): Return count of dup event dig vals at key Returns zero if no entry at key """ - return self.cntIoVals(self.dels, key) + return self.cntIoDupVals(self.dels, key) def delDes(self, key): """ @@ -3232,9 +3124,9 @@ def delDes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.dels, key) + return self.delIoDupVals(self.dels, key) - def getDelIter(self, pre): + def getDelItemIter(self, pre): """ Returns iterator of all dup vals in insertion order for any entries with same prefix across all sequence numbers including gaps. @@ -3251,7 +3143,8 @@ def getDelIter(self, pre): """ if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.getIoValsAnyPreIter(self.dels, pre) + return self.getTopIoDupItemIter(self.dels, pre) + #return self.getOnIoDupValsAnyPreIter(self.dels, pre) def putLdes(self, key, vals): """ @@ -3261,7 +3154,7 @@ def putLdes(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.ldes, key, vals) + return self.putIoDupVals(self.ldes, key, vals) def addLde(self, key, val): """ @@ -3271,7 +3164,7 @@ def addLde(self, key, val): Returns True if written else False if dup val already exists Duplicates are inserted in insertion order. """ - return self.addIoVal(self.ldes, key, val) + return self.addIoDupVal(self.ldes, key, val) def getLdes(self, key): """ @@ -3280,7 +3173,7 @@ def getLdes(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.ldes, key) + return self.getIoDupVals(self.ldes, key) def getLdeLast(self, key): """ @@ -3289,7 +3182,7 @@ def getLdeLast(self, key): Returns None if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoValLast(self.ldes, key) + return self.getIoDupValLast(self.ldes, key) def getLdeItemIter(self, key=b''): """ @@ -3302,30 +3195,7 @@ def getLdeItemIter(self, key=b''): Duplicates are retrieved in insertion order. """ return self.getTopIoDupItemIter(self.ldes, key) - - def getLdeItemsNext(self, key=b'', skip=True): - """ - Use snKey() - Return all dups of likely duplicitous escrowed event dig items at next key after key. - Item is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Returns empty list if no entry at key - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNext(self.ldes, key, skip) - - def getLdeItemsNextIter(self, key=b'', skip=True): - """ - Use sgKey() - Return iterator of likely duplicitous escrowed event dig items at next key after key. - Items is (key, val) where proem has already been stripped from val - If key is b'' empty then returns dup items at first key. - If skip is False and key is not b'' empty then returns dup items at key - Raises StopIteration Error when empty - Duplicates are retrieved in insertion order. - """ - return self.getIoItemsNextIter(self.ldes, key, skip) + #return self.getIoDupItemsNextIter(self.ldes, key, skip) def cntLdes(self, key): """ @@ -3333,7 +3203,7 @@ def cntLdes(self, key): Return count of dup event dig at key Returns zero if no entry at key """ - return self.cntIoVals(self.ldes, key) + return self.cntIoDupVals(self.ldes, key) def delLdes(self, key): """ @@ -3341,7 +3211,7 @@ def delLdes(self, key): Deletes all values at key. Returns True If key exists in database Else False """ - return self.delIoVals(self.ldes, key) + return self.delIoDupVals(self.ldes, key) def delLde(self, key, val): """ @@ -3354,7 +3224,7 @@ def delLde(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.ldes, key, val) + return self.delIoDupVal(self.ldes, key, val) class BaserDoer(doing.Doer): @@ -3409,7 +3279,7 @@ def __init__(self, baser, **kwa): super(BaserDoer, self).__init__(**kwa) self.baser = baser - def enter(self): + def enter(self, *, temp=False): """""" if not self.baser.opened: self.baser.reopen() diff --git a/src/keri/db/dbing.py b/src/keri/db/dbing.py index 3b5b967b8..55ff69a12 100644 --- a/src/keri/db/dbing.py +++ b/src/keri/db/dbing.py @@ -53,22 +53,73 @@ from typing import Union import lmdb -from ordered_set import OrderedSet as oset - -from hio.base import filing - +from ordered_set import OrderedSet as oset from hio.base import filing import keri +from .. import help +from ..kering import MaxON # maximum ordinal number for seqence or first seen from ..help import helping +logger = help.ogler.getLogger() + ProemSize = 32 # does not include trailing separator MaxProem = int("f"*(ProemSize), 16) -MaxON = int("f"*32, 16) # largest possible ordinal number, sequence or first seen - SuffixSize = 32 # does not include trailing separator MaxSuffix = int("f"*(SuffixSize), 16) +KERILMDBMapSizeKey = "KERI_LMDB_MAP_SIZE" # fallback +KERIBaserMapSizeKey = "KERI_BASER_MAP_SIZE" +KERIRegerMapSizeKey = "KERI_REGER_MAP_SIZE" +KERIKeeperMapSizeKey = "KERI_KEEPER_MAP_SIZE" +KERIMailboxerMapSizeKey = "KERI_MAILBOXER_MAP_SIZE" +KERINoterMapSizeKey = "KERI_NOTER_MAP_SIZE" + +def onKey(top, on, *, sep=b'.'): + """ + Returns: + onkey (bytes): key formed by joining top key and hex str conversion of + int ordinal number on with sep character. + + Parameters: + top (str | bytes): top key prefix to be joined with hex version of on using sep + on (int): ordinal number to be converted to 32 hex bytes + sep (bytes): separator character for join + + """ + if hasattr(top, "encode"): + top = top.encode("utf-8") # convert str to bytes + return (b'%s%s%032x' % (top, sep, on)) + + +def snKey(pre, sn): + """ + Returns: + snkey (bytes): key formed by joining pre and hex str conversion of int + sequence ordinal number sn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + sn (int): sequence number to be converted to 32 hex bytes + """ + return onKey(pre, sn, sep=b'.') + + +def fnKey(pre, fn): + """ + Returns: + fnkey (bytes): key formed by joining pre and hex str conversion of int + first seen ordinal number fn with sep character b".". + + Parameters: + pre (str | bytes): key prefix to be joined with hex version of on using + b"." sep + fn (int): first seen ordinal number to be converted to 32 hex bytes + """ + return onKey(pre, fn, sep=b'.') + + def dgKey(pre, dig): """ Returns bytes DB key from concatenation of '.' with qualified Base64 prefix @@ -82,20 +133,6 @@ def dgKey(pre, dig): return (b'%s.%s' % (pre, dig)) -def onKey(pre, sn, *, sep=b'.'): - """ - Returns bytes DB key from concatenation with '.' of qualified Base64 prefix - bytes pre and int ordinal number of event, such as sequence number or first - seen order number. - """ - if hasattr(pre, "encode"): - pre = pre.encode("utf-8") # convert str to bytes - return (b'%s%s%032x' % (pre, sep, sn)) - -snKey = onKey # alias so intent is clear, sn vs fn -fnKey = onKey # alias so intent is clear, sn vs fn - - def dtKey(pre, dts): """ Returns bytes DB key from concatenation of '|' qualified Base64 prefix @@ -111,7 +148,8 @@ def dtKey(pre, dts): dts = dts.encode("utf-8") # convert str to bytes return (b'%s|%s' % (pre, dts)) - +# ToDo right split so key prefix could be top of key space with more than one +# part def splitKey(key, sep=b'.'): """ Returns duple of pre and either dig or on, sn, fn str or dts datetime str by @@ -131,13 +169,13 @@ def splitKey(key, sep=b'.'): else: if hasattr(sep, 'encode'): # make sep match bytes or str sep = sep.encode("utf-8") - splits = key.split(sep) + splits = key.rsplit(sep, 1) if len(splits) != 2: - raise ValueError("Unsplittable key = {}".format(key)) + raise ValueError(f"Unsplittable {key=} at {sep=}.") return tuple(splits) -def splitKeyON(key, *, sep=b'.'): +def splitOnKey(key, *, sep=b'.'): """ Returns list of pre and int on from key Accepts either bytes or str key @@ -149,11 +187,13 @@ def splitKeyON(key, *, sep=b'.'): on = int(on, 16) return (top, on) -splitSnKey = splitKeyON # alias so intent is clear, sn vs fn; backport of 1.2.x alias -splitFnKey = splitKeyON # alias so intent is clear, sn vs fn; backport of 1.2.x alias -splitKeySN = splitKeyON # alias so intent is clear, sn vs fn -splitKeyFN = splitKeyON # alias so intent is clear, sn vs fn +splitSnKey = splitOnKey # alias so intent is clear, sn vs fn +splitFnKey = splitOnKey # alias so intent is clear, sn vs fn + +splitKeyON = splitOnKey # backwards compatible alias +splitKeySN = splitSnKey # backwards compatible alias +splitKeyFN = splitFnKey # backwards compatible alias def splitKeyDT(key): @@ -312,7 +352,6 @@ class LMDBer(filing.Filer): MaxNamedDBs = 96 MapSize = 104857600 - def __init__(self, readonly=False, **kwa): """ Setup main database directory at .dirpath. @@ -349,11 +388,12 @@ def __init__(self, readonly=False, **kwa): False means open database in read/write mode """ + self.env = None self._version = None self.readonly = True if readonly else False - super(LMDBer, self).__init__(**kwa) + super(LMDBer, self).__init__(**kwa) def reopen(self, readonly=False, **kwa): """ @@ -387,12 +427,7 @@ def reopen(self, readonly=False, **kwa): # open lmdb major database instance # creates files data.mdb and lock.mdb in .dbDirPath - map_size = os.getenv("KERI_LMDB_MAP_SIZE", '4294967296') # 4GB - try: - map_size = int(map_size) - except ValueError: - map_size = 4 * 1024**3 # 4GB - self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=map_size, + self.env = lmdb.open(self.path, max_dbs=self.MaxNamedDBs, map_size=self.MapSize, mode=self.perm, readonly=self.readonly) self.opened = True if opened and self.env else False @@ -402,8 +437,6 @@ def reopen(self, readonly=False, **kwa): return self.opened - - @property def version(self): """ Return the version of database stored in __version__ key. @@ -447,7 +480,7 @@ def close(self, clear=False): self.env = None - return (super(LMDBer, self).close(clear=clear)) + return super(LMDBer, self).close(clear=clear) def getVer(self): """ Returns the value of the the semver formatted version in the __version__ key in this database @@ -475,7 +508,6 @@ def setVer(self, val): cursor = txn.cursor() cursor.replace(b'__version__', val) - # For subdbs with no duplicate values allowed at each key. (dupsort==False) def putVal(self, db, key, val): """ @@ -501,7 +533,9 @@ def setVal(self, db, key, val): """ Write serialized bytes val to location key in db Overwrites existing val if any - Returns True If val successfully written Else False + Returns: + result (bool): True If val successfully written + False otherwise Parameters: db is opened named sub db with dupsort=False @@ -566,41 +600,9 @@ def cnt(self, db): return count - def getAllItemIter(self, db, key=b'', split=True, sep=b'.'): - """ - Returns iterator of item duple (key, val), at each key over all - keys in db. If split is true then the key is split at sep and instead - of returing duple it results tuple with one entry for each key split - as well as the value. - - Works for both dupsort==False and dupsort==True - - Raises StopIteration Error when empty. - - Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database - split (bool): True means split key at sep before returning - sep (bytes): separator char for key - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db - - for key, val in cursor.iternext(): # return key, val at cursor - if split: - splits = bytes(key).split(sep) - splits.append(val) - else: - splits = (bytes(key), val) - yield tuple(splits) - - - def getTopItemIter(self, db, key=b''): + def getTopItemIter(self, db, top=b''): """ - Iterates over branch of db given by key + Iterates over branch of db given by top key Returns: items (abc.Iterator): iterator of (full key, val) tuples over a @@ -615,22 +617,24 @@ def getTopItemIter(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then gets all items in database. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any for ckey, cval in cursor.iternext(): # get key, val at cursor ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done yield (ckey, cval) # another entry in branch startswith key return # done raises StopIteration - def delTopVal(self, db, key=b''): + def delTopVal(self, db, top=b''): """ Deletes all values in branch of db given top key. @@ -640,9 +644,9 @@ def delTopVal(self, db, key=b''): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): truncated top key, a key space prefix to get all the items + top (bytes): truncated top key, a key space prefix to get all the items from multiple branches of the key space. If top key is - empty then gets all items in database + empty then deletes all items in database Works for both dupsort==False and dupsort==True Because cursor.iternext() advances cursor after returning item its safe @@ -656,188 +660,301 @@ def delTopVal(self, db, key=b''): with self.env.begin(db=db, write=True, buffers=True) as txn: result = False cursor = txn.cursor() - if cursor.set_range(key): # move to val at key >= key if any + if cursor.set_range(top): # move to val at key >= key if any ckey, cval = cursor.item() while ckey: # end of database key == b'' ckey = bytes(ckey) - if not ckey.startswith(key): # prev entry if any last in branch + if not ckey.startswith(top): # prev entry if any last in branch break # done result = cursor.delete() or result # delete moves cursor to next item ckey, cval = cursor.item() # cursor now at next item after deleted return result - # ported from OnIoDupSuber - def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): - """ - Returns iterator of triples (key, on, val), at each key over all ordinal - numbered keys with same key + sep + on in db. Values are sorted by - onKey(key, on) where on is ordinal number int and key is prefix sans on. - Values duplicates are sorted internally by hidden prefixed insertion order - proem ordinal - Returned items are triples of (key, on, val) - when key is empty then retrieves whole db - Raises StopIteration Error when empty. + # For subdbs the use keys with trailing part the is monotonically + # ordinal number serialized as 32 hex bytes + + # used in OnSuberBase + def putOnVal(self, db, key, on=0, val=b'', *, sep=b'.'): + """Write serialized bytes val to location at onkey consisting of + key + sep + serialized on in db. + Does not overwrite. Returns: - items (Iterator[(key, on, val)]): triples of key, on, val + result (bool): True if successful write i.e onkey not already in db + False otherwise Parameters: - db (subdb): named sub db in lmdb + db (lmdbsubdb): named sub db of lmdb key (bytes): key within sub db's keyspace plus trailing part on - when key is empty then retrieves whole db - on (int): ordinal number at which to initiate retrieval + on (int): ordinal number at which write + val (bytes): to be written at onkey sep (bytes): separator character for split """ - for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): - val = val[33:] # strip proem - yield (key, on, val) + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.put(onkey, val, overwrite=False)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - # ported from OnSuberBase - def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): + # used in OnSuberBase + def setOnVal(self, db, key, on=0, val=b'', *, sep=b'.'): """ - Returns iterator of triples (key, on, val), at each key over all ordinal - numbered keys with same key + sep + on in db. Values are sorted by - onKey(key, on) where on is ordinal number int and key is prefix sans on. - Returned items are triples of (key, on, val) - When dupsort==true then duplicates are included in items since .iternext - includes duplicates. - when key is empty then retrieves whole db - - Raises StopIteration Error when empty. + Write serialized bytes val to location at onkey consisting of + key + sep + serialized on in db. + Overwrites pre-existing value at onkey if any. Returns: - items (Iterator[(key, on, val)]): triples of key, on, val with same - key but increments of on beginning with on + result (bool): True if successful write i.e onkey not already in db + False otherwise Parameters: - db (subdb): named sub db in lmdb + db (lmdbsubdb): named sub db of lmdb key (bytes): key within sub db's keyspace plus trailing part on - when key is empty then retrieves whole db - on (int): ordinal number at which to initiate retrieval + on (int): ordinal number at which write + val (bytes): to be written at onkey sep (bytes): separator character for split """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() + with self.env.begin(db=db, write=True, buffers=True) as txn: if key: # not empty onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest - else: # empty + else: onkey = key - if not cursor.set_range(onkey): # moves to val at key >= onkey - return # no values end of db raises StopIteration + try: + return (txn.put(onkey, val)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - for ckey, cval in cursor.iternext(): # get key, val at cursor - ckey, cn = splitKeyON(ckey, sep=sep) - if key and not ckey == key: - break - yield (ckey, cn, cval) - # For subdbs with no duplicate values allowed at each key. (dupsort==False) - # and use keys with ordinal as monotonically increasing number part - # such as sn or fn - def appendOrdValPre(self, db, pre, val): + # used in OnSuberBase + def appendOnVal(self, db, key, val, *, sep=b'.'): """ - Appends val in order after last previous key with same pre in db. - Returns ordinal number in, on, of appended entry. Appended on is 1 greater - than previous latest on. - Uses onKey(pre, on) for entries. + Appends val in order after last previous onkey in db where + onkey has same given key prefix but with different serialized on suffix + attached with sep. + Returns ordinal number on, of appended entry. Appended on is 1 greater + than previous latest on at key. + Uses onKey(key, on) for entries. - Append val to end of db entries with same pre but with on incremented by - 1 relative to last preexisting entry at pre. + Works with either dupsort==True or False since always creates new full + key. + + Append val to end of db entries with same key but with on incremented by + 1 relative to last preexisting entry at key. + + Returns: + on (int): ordinal number of newly appended val Parameters: - db is opened named sub db with dupsort=False - pre is bytes identifier prefix for event - val is event digest + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split """ # set key with fn at max and then walk backwards to find last entry at pre # if any otherwise zeroth entry at pre - key = onKey(pre, MaxON) + onkey = onKey(key, MaxON, sep=sep) with self.env.begin(db=db, write=True, buffers=True) as txn: on = 0 # unless other cases match then zeroth entry at pre cursor = txn.cursor() - if not cursor.set_range(key): # max is past end of database + if not cursor.set_range(onkey): # max is past end of database # so either empty database or last is earlier pre or # last is last entry at same pre if cursor.last(): # not empty db. last entry earlier than max - ckey = cursor.key() - cpre, cn = splitKeyON(ckey) - if cpre == pre: # last is last entry for same pre + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) + if ckey == key: # last is last entry for same pre on = cn + 1 # increment else: # not past end so not empty either later pre or max entry at pre - ckey = cursor.key() - cpre, cn = splitKeyON(ckey) - if cpre == pre: # last entry for pre is already at max - raise ValueError("Number part of key {} exceeds maximum" - " size.".format(ckey)) + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) + if ckey == key: # last entry for pre is already at max + raise ValueError(f"Number part {cn=} for key part {ckey=}" + f"exceeds maximum size.") else: # later pre so backup one entry # either no entry before last or earlier pre with entry if cursor.prev(): # prev entry, maybe same or earlier pre - ckey = cursor.key() - cpre, cn = splitKeyON(ckey) - if cpre == pre: # last entry at pre + onkey = cursor.key() + ckey, cn = splitOnKey(onkey, sep=sep) + if ckey == key: # last entry at pre on = cn + 1 # increment - key = onKey(pre, on) + onkey = onKey(key, on, sep=sep) - if not cursor.put(key, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) + if not cursor.put(onkey, val, overwrite=False): + raise ValueError(f"Failed appending {val=} at {key=}.") return on - def getAllOrdItemPreIter(self, db, pre, on=0): + # used in OnSuberBase + def getOnVal(self, db, key, on=0, *, sep=b'.'): + """Gets value at onkey consisting of key + sep + serialized on in db. + + Returns: + val (bytes | memoryview): entry at onkey consisting of key + sep + + serialized on in db. + None if no entry at key + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which to retrieve + sep (bytes): separator character for split + """ - Returns iterator of duple item, (on, dig), at each key over all ordinal - numbered keys with same prefix, pre, in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Returned items are duples of (on, dig) where on is ordinal number int - and dig is event digest for lookup in .evts sub db. + with self.env.begin(db=db, write=False, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return(txn.get(onkey)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{onkey}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - Raises StopIteration Error when empty. + + # used in OnSuberBase + def delOnVal(self, db, key, on=0, *, sep=b'.'): + """ + Deletes value at onkey consisting of key + sep + serialized on in db. + Returns True If key exists in database Else False Parameters: - db is opened named sub db with dupsort=False - pre is bytes of itdentifier prefix - on is int ordinal number to resume replay + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + on (int): ordinal number at which to delete + sep (bytes): separator character for split + """ + with self.env.begin(db=db, write=True, buffers=True) as txn: + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + try: + return (txn.delete(onkey)) # when empty deletes whole db + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + + # used in OnSuberBase + def cntOnVals(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns (int): count of of all ordinal keyed vals with key + but different on tail in db starting at ordinal number on of key. + Full key is composed of top+sep+ + When dupsort==true then duplicates are included in count since .iternext + includes duplicates. + when key is empty then counts whole db + + Parameters: + db (lmdbsubdb): named sub db of lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate count + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - if not cursor.set_range(key): # moves to val at key >= key - return # no values end of db + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: + onkey = key + count = 0 + if not cursor.set_range(onkey): # moves to val at key >= key + return count # no values end of db + + for ckey in cursor.iternext(values=False): # get key only at cursor + try: + ckey, cn = splitOnKey(ckey, sep=sep) + except ValueError as ex: # not splittable key + break - for key, val in cursor.iternext(): # get key, val at cursor - cpre, cn = splitKeyON(key) - if cpre != pre: # prev is now the last event for pre + if key and ckey != key: # prev is now the last event for pre break # done - yield (cn, val) # (on, dig) of event + count = count+1 + return count - def getAllOrdItemAllPreIter(self, db, key=b''): + # used in OnSuberBase + def getOnValIter(self, db, key=b'', on=0, *, sep=b'.'): """ - Returns iterator of triple item, (pre, on, dig), at each key over all - ordinal numbered keys for all prefixes in db. Values are sorted by - onKey(pre, on) where on is ordinal number int. - Each returned item is triple (pre, on, dig) where pre is identifier prefix, - on is ordinal number int and dig is event digest for lookup in .evts sub db. + Returns iterator of the val at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db Raises StopIteration Error when empty. + Returns: + items (Iterator[bytes]): val with same + key but increments of on beginning with on + Parameters: - db is opened named sub db with dupsort=False - key is key location in db to resume replay, - If empty then start at first key in database + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for (key, on, val) in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + # used in OnSuberBase + def getOnItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on beginning with on + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - if not cursor.set_range(key): # moves to val at key >= key, first if empty - return # no values end of db + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + if not cursor.set_range(onkey): # moves to val at key >= onkey + return # no values end of db raises StopIteration + + for ckey, cval in cursor.iternext(): # get key, val at cursor + ckey, cn = splitOnKey(ckey, sep=sep) + if key and not ckey == key: + break + yield (ckey, cn, cval) - for key, val in cursor.iternext(): # return key, val at cursor - cpre, cn = splitKeyON(key) - yield (cpre, cn, val) # (pre, on, dig) of event + # ToDo + # getOnItemBackIter symmetric with getOnItemIter + # getOnValBackIter symmetric with getOnValIter + # IoSet insertion order in val so can have effective dups but with + # dupsort==False so val not limited to 511 bytes # For databases that support set of insertion ordered values with apparent # effective duplicate key but with (dupsort==False). Actual key uses hidden # key suffix ordinal to provide insertion ordering of value members of set @@ -860,7 +977,7 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): Parameters: db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): Apparent effective key - vals (abc.Iterable): serialized values to add to set of vals at key + vals (Iterable): serialized values to add to set of vals at key """ result = False @@ -891,8 +1008,8 @@ def putIoSetVals(self, db, key, vals, *, sep=b'.'): def addIoSetVal(self, db, key, val, *, sep=b'.'): """ - Add val to insertion ordered set of values all with the same apparent - effective key if val not already in set of vals at key. + Add val idempotently to insertion ordered set of values all with the + same apparent effective key if val not already in set of vals at key. A Uses hidden ordinal key suffix for insertion ordering. The suffix is appended and stripped transparently. @@ -951,72 +1068,20 @@ def setIoSetVals(self, db, key, vals, *, sep=b'.'): return result - def appendIoSetVal(self, db, key, val, *, sep=b'.'): - """ - Append val to insertion ordered set of values all with the same apparent - effective key. Assumes val is not already in set. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. + def getIoSetVals(self, db, key, *, ion=0, sep=b'.'): + """ Returns: - ion (int): hidden insertion ordering ordinal of appended val + ioset (oset): the insertion ordered set of values at same apparent + effective key. + Uses hidden ordinal key suffix for insertion ordering. + The suffix is appended and stripped transparently. Parameters: db (lmdb._Database): instance of named sub db with dupsort==False key (bytes): Apparent effective key - val (bytes): value to append - """ - ion = 0 # default is zeroth insertion at key - iokey = suffix(key, ion=MaxSuffix, sep=sep) # make iokey at max and walk back - with self.env.begin(db=db, write=True, buffers=True) as txn: - cursor = txn.cursor() # create cursor to walk back - if not cursor.set_range(iokey): # max is past end of database - # Three possibilities for max past end of database - # 1. last entry in db is for same key - # 2. last entry in db is for other key before key - # 3. database is empty - if cursor.last(): # not 3. empty db, so either 1. or 2. - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last is last entry for same key - ion = cion + 1 # so set ion to the increment of cion - else: # max is not past end of database - # Two possibilities for max not past end of databseso - # 1. cursor at max entry at key - # 2. other key after key with entry in database - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # 1. last entry for key is already at max - raise ValueError("Number part of key {} at maximum" - " size.".format(ckey)) - else: # 2. other key after key so backup one entry - # Two possibilities: 1. no prior entry 2. prior entry - if cursor.prev(): # prev entry, maybe same or earlier pre - # 2. prior entry with two possiblities: - # 1. same key - # 2. other key before key - ckey, cion = unsuffix(cursor.key(), sep=sep) - if ckey == key: # prior (last) entry at key - ion = cion + 1 # so set ion to the increment of cion - - iokey = suffix(key, ion=ion, sep=sep) - if not cursor.put(iokey, val, overwrite=False): - raise ValueError("Failed appending {} at {}.".format(val, key)) - - return ion - - - def getIoSetVals(self, db, key, *, ion=0, sep=b'.'): - """ - Returns: - ioset (oset): the insertion ordered set of values at same apparent - effective key. - Uses hidden ordinal key suffix for insertion ordering. - The suffix is appended and stripped transparently. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - ion (int): starting ordinal value, default 0 - + ion (int): starting ordinal value, default 0 + """ with self.env.begin(db=db, write=False, buffers=True) as txn: vals = [] @@ -1196,78 +1261,30 @@ def delIoSetVal(self, db, key, val, *, sep=b'.'): return False - def getIoSetItems(self, db, key, *, ion=0, sep=b'.'): + def getTopIoSetItemIter(self, db, top=b'', *, sep=b'.'): """ Returns: - items (list): list of tuples (iokey, val) of entries in set of with - same apparent effective key. iokey includes the ordinal key suffix - Uses hidden ordinal key suffix for insertion ordering. - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - ion (int): starting ordinal value, default 0 - - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - items = [] - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get iokey, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - items.append((iokey, val)) # another entry at key - return items - + items (Iterator[(key,val)]): iterator of tuples (key, val) where + key is apparent key with hidden insertion ordering suffixe removed + from effective key. + Iterates over top branch of insertion ordered set values where each + effective key has trailing hidden suffix of serialization of insertion + ordering ordinal. - def getIoSetItemsIter(self, db, key, *, ion=0, sep=b'.'): - """ - Returns: - items (abc.Iterator): iterator over insertion ordered set of values - at same apparent effective key where each iteration returns tuple - (iokey, val). iokey includes the ordinal key suffix. Uses hidden ordinal key suffix for insertion ordering. Raises StopIteration Error when empty. Parameters: db (lmdb._Database): instance of named sub db with dupsort==False - key (bytes): Apparent effective key - ion (int): starting ordinal value, default 0 + top (bytes): top key in db. When top is empty then every item in db. + sep (bytes): sep character for attached io suffix """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - iokey = suffix(key, ion, sep=sep) # start ion th value for key zeroth default - cursor = txn.cursor() - if cursor.set_range(iokey): # move to val at key >= iokey if any - for iokey, val in cursor.iternext(): # get key, val at cursor - ckey, cion = unsuffix(iokey, sep=sep) - if ckey != key: # prev entry if any was the last entry for key - break # done - yield (iokey, val) # another entry at key - return # done raises StopIteration + for iokey, val in self.getTopItemIter(db=db, top=top): + key, ion = splitOnKey(iokey, sep=sep) + yield (key, val) - def delIoSetIokey(self, db, iokey): - """ - Deletes val at at actual iokey that includes ordinal key suffix. - - Returns: - result (bool): True if val was deleted at iokey. False otherwise - if no val at iokey - - Parameters: - db (lmdb._Database): instance of named sub db with dupsort==False - iokey (bytes): actual key with ordinal key suffix - """ - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return txn.delete(iokey) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{iokey}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - # For subdbs that support duplicates at each key (dupsort==True) def putVals(self, db, key, vals): @@ -1419,31 +1436,6 @@ def cntVals(self, db, key): return count - def cntValsAllPre(self, db, pre, on=0): - """ - Returns (int): count of of all vals with same pre in key but different - on in key in db starting at ordinal number on of pre - - Does not count dups - - Parameters: - db is opened named sub db - pre is bytes of key within sub db's keyspace pre.on - """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = onKey(pre, on) # start replay at this enty 0 is earliest - count = 0 - if not cursor.set_range(key): # moves to val at key >= key - return count # no values end of db - - for val in cursor.iternext(values=False): # get key, val at cursor - cpre, cn = splitKeyON(val) - if cpre != pre: # prev is now the last event for pre - break # done - count = count+1 - - return count def delVals(self, db, key, val=b''): @@ -1466,8 +1458,10 @@ def delVals(self, db, key, val=b''): # For subdbs that support insertion order preserving duplicates at each key. - # dupsort==True and prepends and strips io val proem - def putIoVals(self, db, key, vals): + # IoDup class IoVals IoItems + # dupsort==True and prepends and strips io val proem to each value. + # because dupsort==True values are limited to 511 bytes including proem + def putIoDupVals(self, db, key, vals): """ Write each entry from list of bytes vals to key in db in insertion order Adds to existing values at key if any @@ -1476,11 +1470,13 @@ def putIoVals(self, db, key, vals): Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order + all values that makes lexocographic order that same as insertion order. + Duplicates are ordered as a pair of key plus value so prepending proem to each value changes duplicate ordering. Proem is 33 characters long. With 32 character hex string followed by '.' for essentiall unlimited number of values which will be limited by memory. + With prepended proem ordinal must explicity check for duplicate values before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). @@ -1492,7 +1488,7 @@ def putIoVals(self, db, key, vals): """ result = False - dups = set(self.getIoVals(db, key)) #get preexisting dups if any + dups = set(self.getIoDupVals(db, key)) #get preexisting dups if any with self.env.begin(db=db, write=True, buffers=True) as txn: idx = 0 cursor = txn.cursor() @@ -1513,7 +1509,7 @@ def putIoVals(self, db, key, vals): return result - def addIoVal(self, db, key, val): + def addIoDupVal(self, db, key, val): """ Add val bytes as dup in insertion order to key in db Adds to existing values at key if any @@ -1521,32 +1517,47 @@ def addIoVal(self, db, key, val): Actual value written include prepended proem ordinal Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: db is opened named sub db with dupsort=False key is bytes of key within sub db's keyspace val is bytes of value to be written """ - return self.putIoVals(db, key, [val]) + return self.putIoDupVals(db, key, [val]) - def getIoVals(self, db, key): + def getIoDupVals(self, db, key): """ Return list of duplicate values at key in db in insertion order Returns empty list if no entry at key Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1566,18 +1577,26 @@ def getIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValsIter(self, db, key): + def getIoDupValsIter(self, db, key): """ Return iterator of all duplicate values at key in db in insertion order Raises StopIteration Error when no remaining dup items = empty. Removes prepended proem ordinal from each val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True @@ -1596,13 +1615,26 @@ def getIoValsIter(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoValLast(self, db, key): + def getIoDupValLast(self, db, key): """ Return last added dup value at key in db in insertion order Returns None no entry at key Removes prepended proem ordinal from val before returning Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1621,77 +1653,96 @@ def getIoValLast(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - def getIoItemsNext(self, db, key=b"", skip=True): + def delIoDupVals(self, db, key): """ - Return list of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key == b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Returns empty list if no entries at next key after key + Deletes all values at key in db if key present. + Returns True If key exists - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. - Assumes DB opened with dupsort=True + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). Parameters: db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty string - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass + key is bytes of key within sub db's keyspace """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - items = [] - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - # slice off prepended ordering prefix on value in item - items = [(key, val[33:]) for key, val in cursor.iternext_dup(keys=True)] - return items - - - def getIoItemsNextIter(self, db, key=b"", skip=True): - """ - Return iterator of all dup items at next key after key in db in insertion order. - Item is (key, val) with proem stripped from val stored in db. - If key = b'' then returns list of dup items at first key in db. - If skip is False and key is not empty then returns dup items at key - Raises StopIteration Error when no remaining dup items = empty. + with self.env.begin(db=db, write=True, buffers=True) as txn: + try: + return (txn.delete(key)) + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - If key is empty then gets io items (key, io value) at first key in db - Use the return key from items as next key for next call to function in - order to iterate through the database + def delIoDupVal(self, db, key, val): + """ + Deletes dup io val at key in db. Performs strip search to find match. + Strips proems and then searches. + Returns True if delete else False if val not present Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentially unlimited + number of values which will be limited by memory. + + Does a linear search so not very efficient when not deleting from the front. + This is hack for supporting escrow which needs to delete individual dup. + The problem is that escrow is not fixed buts stuffs gets added and + deleted which just adds to the value of the proem. 2**16 is an impossibly + large number so the proem will not max out practically. But its not + an elegant solution. + Parameters: - db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace or empty - skip is Boolean If True skips to next key if key is not empty string - Othewise don't skip for first pass + db is opened named sub db with dupsort=False + key is bytes of key within sub db's keyspace + val is bytes of value to be deleted without intersion ordering proem """ - with self.env.begin(db=db, write=False, buffers=True) as txn: + with self.env.begin(db=db, write=True, buffers=True) as txn: cursor = txn.cursor() - if cursor.set_range(key): # moves to first_dup at key - found = True - if skip and key and cursor.key() == key: # skip to next key - found = cursor.next_nodup() # skip to next key not dup if any - if found: - for key, val in cursor.iternext_dup(keys=True): - yield (key, val[33:]) # slice off prepended ordering prefix + try: + if cursor.set_key(key): # move to first_dup + for proval in cursor.iternext_dup(): # value with proem + if val == proval[33:]: # strip of proem + return cursor.delete() + except lmdb.BadValsizeError as ex: + raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," + " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + return False - def cntIoVals(self, db, key): + def cntIoDupVals(self, db, key): """ Return count of dup values at key in db, or zero otherwise Assumes DB opened with dupsort=True + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + Parameters: db is opened named sub db with dupsort=True key is bytes of key within sub db's keyspace @@ -1708,6 +1759,8 @@ def cntIoVals(self, db, key): " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") return count + +# used in IoDupSuber.getItemIter def getTopIoDupItemIter(self, db, top=b''): """ Iterates over top branch of db given by key of IoDup items where each value @@ -1749,218 +1802,340 @@ def getTopIoDupItemIter(self, db, top=b''): before insertion. Uses a python set for the duplicate inclusion test. Set inclusion scales with O(1) whereas list inclusion scales with O(n). """ - for top, val in self.getTopItemIter(db=db, key=top): + for top, val in self.getTopItemIter(db=db, top=top): val = val[33:] # strip proem yield (top, val) - def delIoVals(self, db, key): + # methods for OnIoDup that combines IoDup value proem with On ordinal numbered + # trailing prefix + # this is so we do the proem add and strip here not in some higher level class + # like suber + + def addOnIoDupVal(self, db, key, on=0, val=b'', sep=b'.'): """ - Deletes all values at key in db if key present. - Returns True If key exists + Add val bytes as dup at onkey consisting of key + sep + serialized on in db. + Adds to existing values at key if any + Returns True if written else False if dup val already exists + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if duplicate val added at onkey idempotent + False if duplicate val preexists at onkey Parameters: db is opened named sub db with dupsort=True - key is bytes of key within sub db's keyspace + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to add at onkey as dup + sep (bytes): separator character for split """ - - with self.env.begin(db=db, write=True, buffers=True) as txn: - try: - return (txn.delete(key)) - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") + onkey = onKey(key, on, sep=sep) + return (self.addIoDupVal(db, key=onkey, val=val)) - def delIoVal(self, db, key, val): + # used in OnIoDupSuber + def appendOnIoDupVal(self, db, key, val, *, sep=b'.'): """ - Deletes dup io val at key in db. Performs strip search to find match. - Strips proems and then searches. - Returns True if delete else False if val not present - Assumes DB opened with dupsort=True + Appends val in order after last previous key with same pre in db where + full key has key prefix and serialized on suffix attached with sep and + value has ordinal proem prefixed. + Returns ordinal number in, on, of appended entry. Appended on is 1 greater + than previous latest on at pre. + Uses onKey(pre, on) for entries. - Duplicates at a given key preserve insertion order of duplicate. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending proem - to each value changes duplicate ordering. Proem is 33 characters long. - With 32 character hex string followed by '.' for essentially unlimited - number of values which will be limited by memory. + Works with either dupsort==True or False since always creates new full + key. - Does a linear search so not very efficient when not deleting from the front. - This is hack for supporting escrow which needs to delete individual dup. - The problem is that escrow is not fixed buts stuffs gets added and - deleted which just adds to the value of the proem. 2**16 is an impossibly - large number so the proem will not max out practically. But its not - and elegant solution. So maybe escrows need to use a different approach. - But really didn't want to add another database just for escrows. + Append val to end of db entries with same pre but with on incremented by + 1 relative to last preexisting entry at pre. + + Returns: + on (int): ordinal number of newly appended val Parameters: - db is opened named sub db with dupsort=False - key is bytes of key within sub db's keyspace - val is bytes of value to be deleted without intersion ordering proem + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized value to append + sep (bytes): separator character for split """ + val = (b'%032x.' % (0)) + val # prepend ordering proem + return (self.appendOnVal(db=db, key=key, val=val, sep=sep)) - with self.env.begin(db=db, write=True, buffers=True) as txn: - cursor = txn.cursor() - try: - if cursor.set_key(key): # move to first_dup - for proval in cursor.iternext_dup(): # value with proem - if val == proval[33:]: # strip of proem - return cursor.delete() - except lmdb.BadValsizeError as ex: - raise KeyError(f"Key: `{key}` is either empty, too big (for lmdb)," - " or wrong DUPFIXED size. ref) lmdb.BadValsizeError") - return False + def delOnIoDupVals(self, db, key, on=0, sep=b'.'): + """Deletes all dup iovals at onkey consisting of key + sep + serialized + on in db. + + Assumes DB opened with dupsort=True + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. + + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if onkey present so all dups at onkey deleted + False if onkey not present - def getIoValsAllPreIter(self, db, pre, on=0): + Parameters: + db is opened named sub db with dupsort=True + key (bytes): key within sub db's keyspace plus trailing part on + sep (bytes): separator character for split """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all ordinal numbers in increasing order - without gaps between ordinal numbers - starting with on, default 0. Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + onkey = onKey(key, on, sep=sep) + return (self.delIoDupVals(db, key=onkey)) - Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. + def delOnIoDupVal(self, db, key, on=0, val=b'', sep=b'.'): + """Deletes dup ioval at key onkey consisting of key + sep + serialized + on in db. + Returns True if deleted else False if dup val not present + Assumes DB opened with dupsort=True + + Duplicates are inserted in lexocographic order not insertion order. + Lmdb does not insert a duplicate unless it is a unique value for that + key. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Does inclusion test to dectect of duplicate already exists + Uses a python set for the duplicate inclusion test. Set inclusion scales + with O(1) whereas list inclusion scales with O(n). + + Returns: + result (bool): True if duplicate val found and deleted + False if duplicate val does not exist at onkey Parameters: db is opened named sub db with dupsort=True - pre (bytes | str): of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to begin iteration at + key (bytes): key within sub db's keyspace plus trailing part on + val (bytes): serialized dup value to del at onkey + sep (bytes): separator character for split """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt+1) - - - def getIoValsAllPreBackIter(self, db, pre, on=0): - """ - Returns iterator of all dup vals in insertion order for all entries - with same prefix across all sequence numbers in decreasing order without gaps - between ordinals at a given pre. - Starting with on (default = 0) as begining ordinal number or sequence number. - Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + onkey = onKey(key, on, sep=sep) + return (self.delIoDupVal(db, key=onkey, val=val)) + + + + # used in OnIoDupSuber + def getOnIoDupValIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of val at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + When dupsort==true then duplicates are included in items since .iternext + includes duplicates. + when key is empty then retrieves whole db Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupItemIter(db=db, key=key, on=on, sep=sep): + yield (val) + + + # used in OnIoDupSuber + def getOnIoDupItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """ + Returns iterator of triples (key, on, val), at each key over all ordinal + numbered keys with same key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of identifier prefix prepended to sn in key - within sub db's keyspace - on (int): is ordinal number to begin iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ - with self.env.begin(db=db, write=False, buffers=True) as txn: - cursor = txn.cursor() - key = snKey(pre, cnt := on) - # set_key returns True if exact key else false - while cursor.set_key(key): # moves to first_dup if valid key - for val in cursor.iternext_dup(): - # slice off prepended ordering prefix - yield val[33:] - key = snKey(pre, cnt:=cnt-1) - - - def getIoValLastAllPreIter(self, db, pre, on=0): - """ - Returns iterator of last only of dup vals of each key in insertion order - for all entries with same prefix across all sequence numbers in increasing order - without gaps starting with on (default = 0). Stops if gap or different pre. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + for key, on, val in self.getOnItemIter(db=db, key=key, on=on, sep=sep): + val = val[33:] # strip proem + yield (key, on, val) + + + def getOnIoDupLastValIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of val of last insertion ordered duplicate at each + key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by onKey(key, on) where on + is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + + when key is empty then retrieves whole db Raises StopIteration Error when empty. + Returns: + val (Iterator[bytes]): last dup val at each onkey - Duplicates are retrieved in insertion order. + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split + """ + for key, on, val in self.getOnIoDupLastItemIter(db=db, key=key, on=on, sep=sep): + yield (val) - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + def getOnIoDupLastItemIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator of triples (key, on, val), of last insertion ordered + duplicate at each key over all ordinal numbered keys with same full key + of key + sep + on in db. Values are sorted by + onKey(key, on) where on is ordinal number int and key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Returned items are triples of (key, on, val) + + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): ordinal number to being iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_key(key): # moves to first_dup - if cursor.last_dup(): # move to last_dup - yield cursor.value()[33:] # slice off prepended ordering prefix - key = snKey(pre, cnt:=cnt+1) + if key: # not empty + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + else: # empty + onkey = key + + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration + + while cursor.last_dup(): # move to last_dup at current ckey + onkey, cval = cursor.item() # get ckey cval of last dup + ckey, on = splitOnKey(onkey, sep=sep) # get key on + if key and not ckey == key: + break + + yield (ckey, on, cval[33:]) # slice off prepended ordering proem + onkey = onKey(ckey, on+1) + if not cursor.set_range(onkey): # # moves to first_dup at key>=onkey + return # no values end of db raises StopIteration - def getIoValsAnyPreIter(self, db, pre, on=0): + def getOnIoDupValBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of values, + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. + + Returned items are vals + + when key is empty then retrieves whole db + + Raises StopIteration Error when empty. + + Returns: + val (Iterator[bytes]): at key including duplicates in backwards order + + Parameters: + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ - Returns iterator of all dup vals in insertion order for any entries - with same prefix across all ordinal numbers in order including gaps - between ordinals at a given pre. Staring with on (default = 0). - Stops when pre is different. + for key, on, val in self.getOnIoDupItemBackIter(db=db, key=key, on=on, sep=sep): + yield (val) - Duplicates that may be deleted such as duplicitous event logs need - to be able to iterate across gaps in ordinal number. - Assumes that key is combination of prefix and sequence number given - by .snKey(). - Removes prepended proem ordinal from each val before returning + def getOnIoDupItemBackIter(self, db, key=b'', on=0, *, sep=b'.'): + """Returns iterator going backwards of triples (key, on, val), + of insertion ordered item at each key over all ordinal numbered keys + with same full key of key + sep + on in db. + Values are sorted by onKey(key, on) where on is ordinal number int and + key is prefix sans on. + Values duplicates are sorted internally by hidden prefixed insertion order + proem ordinal + Backwards means decreasing numerical value of duplicate proem, for each on, + decreasing numerical value on for each key and decresing lexocogrphic + order of each key prefix. + + Returned items are triples of (key, on, val) + + when key is empty then retrieves whole db Raises StopIteration Error when empty. - Duplicates are retrieved in insertion order. - Because lmdb is lexocographic an insertion ordering proem is prepended to - all values that makes lexocographic order that same as insertion order - Duplicates are ordered as a pair of key plus value so prepending prefix - to each value changes duplicate ordering. Proem is 17 characters long. - With 16 character hex string followed by '.'. + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val Parameters: - db is opened named sub db with dupsort=True - pre is bytes of itdentifier prefix prepended to sn in key - within sub db's keyspace - on (int): beginning ordinal number to start iteration + db (subdb): named sub db in lmdb + key (bytes): key within sub db's keyspace plus trailing part on + when key is empty then retrieves whole db + on (int): ordinal number at which to initiate retrieval + sep (bytes): separator character for split """ with self.env.begin(db=db, write=False, buffers=True) as txn: cursor = txn.cursor() - key = snKey(pre, cnt:=on) - while cursor.set_range(key): # moves to first dup of key >= key - key = cursor.key() # actual key - front, back = bytes(key).split(sep=b'.', maxsplit=1) - if front != pre: # set range may skip pre if none - break - for val in cursor.iternext_dup(): - yield val[33:] # slice off prepended ordering prefix - cnt = int(back, 16) - key = snKey(pre, cnt:=cnt+1) + if not cursor.last(): # pre-position cursor at last dup of last key + return # empty database so raise StopIteration + + if key: # not empty so attempt to position at starting key not last + onkey = onKey(key, on, sep=sep) # start replay at this enty 0 is earliest + if cursor.set_range(onkey): # found key >= onkey + ckey, cn = splitOnKey(cursor.key(), sep=sep) + if ckey == key: # onkey in db + cursor.last_dup() # start at its last dup + else: # get closest key < onkey + if not cursor.prev(): # last dup of previous key + return # no earlier keys to designated start + + # cursor should now be correctly positioned for start either at + # last dup of either last key or onkey + for onkey, cval in cursor.iterprev(): # iterate backwards + ckey, on = splitOnKey(onkey, sep=sep) + if key and ckey != key: + return + yield (ckey, on, cval[33:]) + + + + # ToDo do we need a replay last backwards? + diff --git a/src/keri/db/escrowing.py b/src/keri/db/escrowing.py index b903373ea..ea820edc5 100644 --- a/src/keri/db/escrowing.py +++ b/src/keri/db/escrowing.py @@ -1,4 +1,4 @@ -1.1# -*- encoding: utf-8 -*- +# -*- encoding: utf-8 -*- """ keri.core.escrowing module @@ -7,13 +7,15 @@ import logging from typing import Type -from keri.core import coring -from keri import help + from keri import kering -from keri.core import eventing -from keri.db import subing +from keri import help from keri.help import helping +from keri.core import coring, eventing, indexing +from keri.db import subing + + logger = help.ogler.getLogger() @@ -42,7 +44,7 @@ def __init__(self, db, subkey, timeout=3600): # given by quadruple (saider.qb64, subkeyer.qb64, seqner.q64, diger.qb64) # of reply and trans signer's key state est evt to val Siger for each # signature. - self.tigerdb = subing.CesrIoSetSuber(db=self.db, subkey=subkey + '-sgs.', klas=coring.Siger) + self.tigerdb = subing.CesrIoSetSuber(db=self.db, subkey=subkey + '-sgs.', klas=indexing.Siger) # all key state kcgs (ksn non-indexed signature serializations) maps ksn SAID # to couple (Verfer, Cigar) of nontrans signer of signature in Cigar @@ -77,7 +79,7 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): extype (Type[Exception]): the expected exception type if the message should remain in escrow """ - for (typ, pre, aid, ion), saider in self.escrowdb.getIoItemIter(keys=(typ,)): + for (typ, pre, aid), saider in self.escrowdb.getItemIter(keys=(typ, '')): try: tsgs = eventing.fetchTsgs(db=self.tigerdb, saider=saider) @@ -88,8 +90,9 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): try: if not (dater and serder and (tsgs or vcigars)): - raise ValueError(f"Missing escrow artifacts at said={saider.qb64}" - f"for pre={pre}.") + msg = f"Missing escrow artifacts at said={saider.qb64} for pre={pre}." + logger.info("Broker %s: unescrow error: %s", typ, msg) + raise ValueError(msg) cigars = [] if vcigars: @@ -101,8 +104,8 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): if ((helping.nowUTC() - dater.datetime) > datetime.timedelta(seconds=self.timeout)): # escrow stale so raise ValidationError which unescrows below - msg = f"{typ} escrow unescrow error: Stale txn state escrow at pre = {pre}" - logger.trace(msg) + msg = f"Escrow unescrow error: Stale txn state escrow at pre = {pre}" + logger.trace("Broker %s: %s", typ, msg) raise kering.ValidationError(msg) processReply(serder=serder, saider=saider, route=serder.ked["r"], @@ -111,29 +114,29 @@ def processEscrowState(self, typ, processReply, extype: Type[Exception]): except extype as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.TRACE): - logger.trace("%s escrow unescrow attempt failed: %s\n", typ, ex.args[0]) - logger.exception("%s escrow unescrow attempt failed: %s\n", typ, ex.args[0]) + logger.trace("Broker %s: unescrow attempt failed: %s\n", typ, ex.args[0]) + logger.exception("Broker %s: unescrow attempt failed: %s", typ, ex.args[0]) except Exception as ex: # other error so remove from reply escrow - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow if logger.isEnabledFor(logging.DEBUG): - logger.exception("%s escrow other error on unescrow: %s\n", typ, ex.args[0]) + logger.exception("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) else: - logger.error("%s escrow other error on unescrow: %s\n", typ, ex.args[0]) + logger.error("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) else: # unescrow succeded - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow only - logger.info("%s escrow unescrow succeeded for txn state = %s", + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow + logger.info("Broker %s: unescrow succeeded for txn state=%s", typ, serder.said) logger.debug("TXN State Body=\n%s\n", serder.pretty()) except Exception as ex: # log diagnostics errors etc - self.escrowdb.remIokey(iokeys=(typ, pre, aid, ion)) # remove escrow + self.escrowdb.rem(keys=(typ, pre, aid), val=saider) # remove escrow self.removeState(saider) if logger.isEnabledFor(logging.DEBUG): - logger.exception("%s escrow unescrowed due to error: %s\n", typ, ex.args[0]) + logger.exception("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) else: - logger.error("%s escrow unescrowed due to error: %s\n", typ, ex.args[0]) + logger.error("Broker %s: unescrowed due to error: %s", typ, ex.args[0]) def escrowStateNotice(self, *, typ, pre, aid, serder, saider, dater, cigars=None, tsgs=None): """ @@ -168,7 +171,7 @@ def escrowStateNotice(self, *, typ, pre, aid, serder, saider, dater, cigars=None for cigar in cigars: # process each couple to verify sig and write to db self.cigardb.put(keys=keys, vals=[(cigar.verfer, cigar)]) - return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # overwrite + return self.escrowdb.put(keys=(typ, pre, aid), vals=[saider]) # does not overwrite def updateReply(self, aid, serder, saider, dater): """ diff --git a/src/keri/db/koming.py b/src/keri/db/koming.py index a4987563a..54a336965 100644 --- a/src/keri/db/koming.py +++ b/src/keri/db/koming.py @@ -47,7 +47,7 @@ class KomerBase: def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, dupsort: bool = False, sep: str = None, **kwa): @@ -73,23 +73,43 @@ def __init__(self, db: dbing.LMDBer, *, self.sep = sep if sep is not None else self.Sep - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable]): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ - Converts key to key str with proper separators and returns key bytes. - If key is already str then returns. Else If key is iterable (non-str) - of strs then joins with separator converts to bytes and returns + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + topive is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key + + Returns: + key (bytes): each element of keys is joined by .sep. If top then last + char of key is also .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ + if hasattr(keys, "encode"): # str + return keys.encode("utf-8") if isinstance(keys, memoryview): # memoryview of bytes return bytes(keys) # return bytes - if hasattr(keys, "encode"): # str - return keys.encode("utf-8") # convert to bytes elif hasattr(keys, "decode"): # bytes - return keys # return as is - return (self.sep.join(keys).encode("utf-8")) # iterable so join + return keys + if topive and keys[-1]: # topive and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) + def _tokeys(self, key: Union[str, bytes, memoryview]): @@ -110,7 +130,34 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): def getItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys. Keys may be keyspace + prefix to return branches of key space. When keys is empty then + returns all items in subdb + + Parameters: + keys (Iterator): tuple of bytes or strs that may be a truncation of + a full keys tuple in in order to get all the items from + multiple branches of the key space. If keys is empty then gets + all items in database. + """ + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): + yield (self._tokeys(key), self.deserializer(val)) + + + def getFullItemIter(self, keys: Union[str, Iterable]=b""): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + Returns: items (Iterator): of (key, val) tuples over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace @@ -124,7 +171,7 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): all items in database. """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, top=self._tokey(keys)): yield (self._tokeys(key), self.deserializer(val)) @@ -133,9 +180,9 @@ def _serializer(self, kind): Parameters: kind (str): serialization """ - if kind == coring.Serials.mgpk: + if kind == coring.Kinds.mgpk: return self.__serializeMGPK - elif kind == coring.Serials.cbor: + elif kind == coring.Kinds.cbor: return self.__serializeCBOR else: return self.__serializeJSON @@ -146,9 +193,9 @@ def _deserializer(self, kind): Parameters: kind (str): deserialization """ - if kind == coring.Serials.mgpk: + if kind == coring.Kinds.mgpk: return self.__deserializeMGPK - elif kind == coring.Serials.cbor: + elif kind == coring.Kinds.cbor: return self.__deserializeCBOR else: return self.__deserializeJSON @@ -218,7 +265,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, **kwa): """ Parameters: @@ -328,7 +375,7 @@ def trim(self, keys: Union[str, Iterable]=b""): Returns: result (bool): True if key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys))) def cntAll(self): @@ -378,7 +425,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'recs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, **kwa): """ Parameters: @@ -549,109 +596,55 @@ def rem(self, keys: Union[str, Iterable], val=None): key=self._tokey(keys), sep=self.sep) + #def remIokey(self, iokeys: str | bytes | memoryview | Iterable): + #""" + #Removes entries at iokeys - def getItemIter(self, keys: Union[str, Iterable]=b""): - """Get items iterator - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. - - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. + #Parameters: + #iokeys (str | bytes | memoryview | Iterable): of key str or + #tuple of key strs to be combined in order to form key - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self.deserializer(val)) + #Returns: + #result (bool): True if key exists so delete successful. False otherwise + #""" + #return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - def getIoSetItem(self, keys: Union[str, Iterable]): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key - Parameters: - keys (Iterable): of key strs to be combined in order to form key + def getItemIter(self, keys: Union[str, Iterable]=b"", *, topive=False): + """Get items iterator Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - - """ - return ([(self._tokeys(iokey), self.deserializer(val)) for iokey, val - in self.db.getIoSetItems(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)]) - - - def getIoSetItemIter(self, keys: Union[str, Iterable]): - """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - - Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done - - """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - sep=self.sep): - yield (self._tokeys(iokey), self.deserializer(val)) - - - def getIoItemIter(self, keys: Union[str, Iterable]=b""): - """ - Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. When keys is empty then returns all items in subdb. - + Returned key in each item has ordinal suffix removed. Parameters: keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + """ + for iokey, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), + sep=self.sep.encode()): yield (self._tokeys(iokey), self.deserializer(val)) - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at iokeys - - Parameters: - iokeys (tuple): of key str or tuple of key strs to be combined in - order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) - - class DupKomer(KomerBase): """ @@ -665,7 +658,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'recs.', schema: Type[dataclass], # class not instance - kind: str = coring.Serials.json, + kind: str = coring.Kinds.json, **kwa): """ Parameters: diff --git a/src/keri/db/migrations/add_key_and_reg_state_schemas.py b/src/keri/db/migrations/add_key_and_reg_state_schemas.py index c626bd2fb..3a632f466 100644 --- a/src/keri/db/migrations/add_key_and_reg_state_schemas.py +++ b/src/keri/db/migrations/add_key_and_reg_state_schemas.py @@ -21,9 +21,8 @@ def _check_if_needed(db): return True def migrate(db): - """Adds schema for KeyStateRecord , RegStateRecord, and migrates the rgy.cancs., hby.db.pubs., + """Adds schema for KeyStateRecord, RegStateRecord, and migrates the rgy.cancs., hby.db.pubs., and hby.db.digs. to be up to date as of 2022-??-?? - This migration performs the following: - hby.db -> "stts." schema from dict -> KeyStateRecord - rgy -> "stts." schema from dict -> RegStateRecord @@ -35,14 +34,13 @@ def migrate(db): "pubs." Verfer of each Verfer for each FEL event "digs." Diger of next Diger (ndiger) of each FEL event Value: (prefix, sn) of each event - Parameters: db(Baser): Baser database object on which to run the migration """ # May be running on a database that is already in the right state yet has no migrations run # so we need to check if the migration is needed if not _check_if_needed(db): - print(f"{__name__} migration not needed, already ran") + print(f"{__name__} migration not needed, database already in correct state") return try: @@ -96,6 +94,7 @@ def migrate(db): b=sad['b'], # list of qb64 may be empty c=sad['c'], ) + # ksr = stateFromKever(kever) rgy.states.pin(sad['i'], val=rsr) for (said,), _ in rgy.saved.getItemIter(): @@ -117,9 +116,7 @@ def migrate(db): db.gpse.trim() db.epse.trim() db.dune.trim() - for ekey, edig in db.getQnfItemsNextIter(): - pre, _ = splitKey(ekey) - db.delQnf(dgKey(pre, edig), edig) + db.qnfs.trim() except ConfigurationError: logger.error(f"identifier prefix for {db.name} does not exist, incept must be run first", ) @@ -135,7 +132,7 @@ def migrateKeys(db): digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.", klas=(coring.Prefixer, coring.Seqner)) - for pre, fn, dig in db.getFelItemAllPreIter(key=b''): + for pre, fn, dig in db.getFelItemAllPreIter(): dgkey = dbing.dgKey(pre, dig) # get message if not (raw := db.getEvt(key=dgkey)): logger.info(f"Migrate keys: missing event for dig={dig}, skipped.") @@ -147,4 +144,4 @@ def migrateKeys(db): pubs.add(keys=(verfer.qb64,), val=val) ndigers = serder.ndigers or [] for diger in ndigers: - digs.add(keys=(diger.qb64,), val=val) + digs.add(keys=(diger.qb64,), val=val) \ No newline at end of file diff --git a/src/keri/db/migrations/hab_data_rename.py b/src/keri/db/migrations/hab_data_rename.py index 7ac814514..09e377966 100644 --- a/src/keri/db/migrations/hab_data_rename.py +++ b/src/keri/db/migrations/hab_data_rename.py @@ -23,13 +23,32 @@ class HabitatRecordV0_6_7: # baser.habs watchers: list[str] = field(default_factory=list) # aids qb64 of watchers +@dataclass +class HabitatRecordV0_6_8: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + def _check_if_needed(db): """ Check if the migration is needed - Parameters: db(Baser): Baser database object on which to run the migration - Returns: bool: True if the migration is needed, False otherwise """ @@ -56,7 +75,7 @@ def migrate(db): # May be running on a database that is already in the right state yet has no migrations run # so we need to check if the migration is needed if not _check_if_needed(db): - print(f"{__name__} migration not needed, already ran") + print(f"{__name__} migration not needed, database already in correct state") return habs = koming.Komer(db=db, @@ -68,7 +87,7 @@ def migrate(db): for name, habord in habs.getItemIter(): existing = asdict(habord) habord_0_6_7 = HabitatRecordV0_6_7(**existing) - habord_0_6_8 = HabitatRecord( + habord_0_6_8 = HabitatRecordV0_6_8( hid=habord_0_6_7.prefix, mid=habord_0_6_7.pid, smids=habord_0_6_7.aids, @@ -76,13 +95,15 @@ def migrate(db): sid=None, watchers=habord_0_6_7.watchers ) - habords[name] = habord_0_6_8 + habords[habord_0_6_8.hid] = habord_0_6_8 habs.trim() # remove existing records # Add in the renamed records - for name, habord in habords.items(): - (name,) = name - db.habs.pin(keys=(name,), val=habord) + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_8, ) + for pre, habord in habords.items(): + habs.pin(keys=(pre,), val=habord) diff --git a/src/keri/db/migrations/rekey_habs.py b/src/keri/db/migrations/rekey_habs.py new file mode 100644 index 000000000..4d84b99b3 --- /dev/null +++ b/src/keri/db/migrations/rekey_habs.py @@ -0,0 +1,91 @@ +from dataclasses import dataclass, field, asdict + +from keri.db import koming, basing + + +@dataclass +class HabitatRecordV0_6_8: # baser.habs + """ + Habitat application state information keyed by habitat name (baser.habs) + + Attributes: + hid (str): identifier prefix of hab qb64 + mid (str | None): group member identifier qb64 when hid is group + smids (list | None): group signing member identifiers qb64 when hid is group + rmids (list | None): group signing member identifiers qb64 when hid is group + watchers: (list[str]) = list of id prefixes qb64 of watchers + + + """ + hid: str # hab own identifier prefix qb64 + mid: str | None = None # group member identifier qb64 when hid is group + smids: list | None = None # group signing member ids when hid is group + rmids: list | None = None # group rotating member ids when hid is group + sid: str | None = None # Signify identifier qb64 when hid is Signify + watchers: list[str] = field(default_factory=list) # id prefixes qb64 of watchers + +def _check_if_needed(db): + habs = koming.Komer(db=db, + subkey='habs.', + schema=dict, ) + first = next(habs.getItemIter(), None) + if first is None: + return False + name, habord = first + if 'domain' in habord: + return False + return True + +def migrate(db): + """ Re-key habs migration for changing the key for .habs and introducing the .names database + + This migrations performs the following: + 1. Rekey .habs from name (alias) to the AID of the Hab + 2. Add Name and domain to the HabitatRecord for all Habs + 3. Populate the .names index as (ns, name) -> AID + 4. Remove the .nmsp namespaced Habs database (replaced within .habs and .names now) + + Parameters: + db(Baser): Baser database object on which to run the migration + """ + # May be running on a database that is already in the right state yet has no migrations run + # so we need to check if the migration is needed + if not _check_if_needed(db): + print(f"{__name__} migration not needed, database already in correct state") + return + + habs = koming.Komer(db=db, + subkey='habs.', + schema=HabitatRecordV0_6_8, ) + + # habitat application state keyed by habitat namespace + b'\x00' + name, includes prefix + nmsp = koming.Komer(db=db, + subkey='nmsp.', + schema=HabitatRecordV0_6_8, ) + + habords = dict() + # Update Hab records from .habs with name + for name, habord in habs.getItemIter(): + name = ".".join(name) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + habords[habord.hid] = nhabord + + habs.trim() + + # Update Hab records from .nmsp with name and domain (ns) + for keys, habord in nmsp.getItemIter(): + ns = keys[0] + name = ".".join(keys[1:]) # detupleize the database key name + nhabord = basing.HabitatRecord(**asdict(habord)) + nhabord.name = name + nhabord.domain = ns + habords[habord.hid] = nhabord + + nmsp.trim() # remove existing records + + # Rekey .habs and create .names index + for pre, habord in habords.items(): + db.habs.pin(keys=(pre,), val=habord) + ns = "" if habord.domain is None else habord.domain + db.names.pin(keys=(ns, habord.name), val=pre) diff --git a/src/keri/db/subing.py b/src/keri/db/subing.py index 76422692c..2daa09311 100644 --- a/src/keri/db/subing.py +++ b/src/keri/db/subing.py @@ -36,13 +36,47 @@ DupSuber CesrDupSuber +Class Architecture + +Suber is simple lexographic database with only one value per key +OnSuber is simple lexographic database where trailing part of key is serialized + ordinal number so that the ordering within each key prefix is monotonically + increasing numeric + +The term 'set' of values means that no value may appear more than once in the set. +Sets support idempotent adds and puts to db. This means one can add or put the same +(key, val) pair multiple times and not change the db. + +DupSuber provides set of lexicographic ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is performant. Good for indices. + +IoDupSuber provides set of insertion ordered values at each key. Each value has + a limited size (key + value <= 511 byes). The set is less perfromant than DupSuber + but more performant than IoSetSuber. Good for insertion ordered indices + +IoSetSuber proves set of insertion ordered values at each key. Value size is not limited + Good for any insertion ordered set where size may be too large for IoDupSuber + +OnIoDupSuber provides set of insertion ordered values where the where trailing + part of key is serialized ordinal number so that the ordering within each + key prefix is monotonically increasing numeric. + +Each of these base types for managing the key space may be mixed with other +Classes that provide different types of values these include. + +Cesr +CatCesr +Serder +etc. + """ from typing import Type, Union from collections.abc import Iterable, Iterator from .. import help -from ..help.helping import nonStringIterable +from ..help.helping import nonStringIterable, Reb64 +from .. import core from ..core import coring, scheming, serdering from . import dbing @@ -59,6 +93,8 @@ class SuberBase(): db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ Sep = '.' # separator for combining key iterables @@ -66,6 +102,7 @@ def __init__(self, db: dbing.LMDBer, *, subkey: str='docs.', dupsort: bool=False, sep: str=None, + verify: bool=False, **kwa): """ Parameters: @@ -76,31 +113,41 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ - super(SuberBase, self).__init__() + super(SuberBase, self).__init__() # for multi inheritance self.db = db self.sdb = self.db.env.open_db(key=subkey.encode("utf-8"), dupsort=dupsort) self.sep = sep if sep is not None else self.Sep + self.verify = True if verify else False + - def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], - top: bool=False): + def _tokey(self, keys: str|bytes|memoryview|Iterable[str|bytes|memoryview], + topive: bool=False): """ - Converts keys to key str with proper separators and returns key bytes. - If keys is already str then returns. Else If keys is iterable (non-str) - of strs then joins with separator converts to bytes and returns. - top allows partial key from top branch of key space given by partial keys + Converts keys to key bytes with proper separators and returns key bytes. + If keys is already str or bytes or memoryview then returns key bytes. + Else If keys is iterable (non-str) of strs or bytes then joins with + separator converts to key bytes and returns. When keys is iterable and + topive is True then enables partial key from top branch of key space given + by partial keys by appending separator to end of partial key Returns: - key (bytes): each element of keys is joined by .sep. If top then last - char of key is also .sep + key (bytes): each element of keys is joined by .sep. If topive then + last char of key is .sep Parameters: - keys (Union[str, bytes, Iterable]): str, bytes, or Iterable of str. - top (bool): True means treat as partial key tuple from top branch of - key space given by partial keys. Resultant key ends in .sep. + keys (str | bytes | memoryview | Iterable[str | bytes]): db key or + Iterable of (str | bytes) to form key. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. False means treat as full branch in key space. Resultant key - does not end in .sep + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of topive value """ if hasattr(keys, "encode"): # str @@ -109,19 +156,22 @@ def _tokey(self, keys: Union[str, bytes, memoryview, Iterable], return bytes(keys) # return bytes elif hasattr(keys, "decode"): # bytes return keys - return (self.sep.join(keys).encode("utf-8")) + if topive and keys[-1]: # topive and keys is not already partial + keys = tuple(keys) + ('',) # cat empty str so join adds trailing sep + return (self.sep.join(key.decode() if hasattr(key, "decode") else key + for key in keys).encode("utf-8")) - def _tokeys(self, key: Union[str, bytes, memoryview]): + def _tokeys(self, key: str | bytes | memoryview): """ Converts key bytes to keys tuple of strs by decoding and then splitting - at separator. + at separator .sep. Returns: - keys (iterable): of str + keys (tuple[str]): makes tuple by splitting key at sep Parameters: - key (Union[str, bytes]): str or bytes. + key (str | bytes | memoryview): db key. """ if isinstance(key, memoryview): # memoryview of bytes @@ -131,61 +181,138 @@ def _tokeys(self, key: Union[str, bytes, memoryview]): return tuple(key.split(self.sep)) - def _ser(self, val: Union[str, memoryview, bytes]): + def _ser(self, val: str | bytes | memoryview): """ Serialize value to bytes to store in db Parameters: - val (Union[str, memoryview, bytes]): encodable as bytes + val (str | bytes | memoryview): encodable as bytes """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # return bytes return (val.encode("utf-8") if hasattr(val, "encode") else val) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: bytes | memoryview): """ Deserialize val to str Parameters: - val (Union[str, memoryview, bytes]): decodable as str + val (bytes | memoryview): decodable as str """ if isinstance(val, memoryview): # memoryview is always bytes val = bytes(val) # convert to bytes return (val.decode("utf-8") if hasattr(val, "decode") else val) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def trim(self, keys: str|bytes|memoryview|Iterable=b"", *, topive=False): + """ + Removes all entries whose keys startswith keys. Enables removal of whole + branches of db key space. To ensure that proper separation of a branch + include empty string as last key in keys. For example ("a","") deletes + 'a.1'and 'a.2' but not 'ab' + + Parameters: + keys (Iteratabke[str | bytes | memoryview]): of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then trims all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + + Returns: + result (bool): True if val at key exists so delete successful. False otherwise """ + return(self.db.delTopVal(db=self.sdb, top=self._tokey(keys, topive=topive))) + + + def getFullItemIter(self, keys: str|bytes|memoryview|Iterable[str|bytes]="", + *, topive=False): + """Iterator over items in .db that returns full items with subclass + specific special hidden parts shown for debugging or testing. + Returns: - items (Iterator): if (key, val) tuples over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb. + This is meant to return full parts of items in both keyspace and + valuespace which may be useful in debugging or testing. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + keys (str|bytes|memoryview|Iteratable[str | bytes | memoryview]): + of key parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): yield (self._tokeys(key), self._des(val)) - def trim(self, keys: Union[str, Iterable]=b""): - """ - Removes all entries whose keys startswith keys. Enables removal of whole - branches of db key space. To ensure that proper separation of a branch - include empty string as last key in keys. For example ("a","") deletes - 'a.1'and 'a.2' but not 'ab' + def getItemIter(self, keys: str|bytes|memoryview|Iterable="", + *, topive=False): + """Iterator over items in .db subclasses that do special hidden transforms + on either the keyspace or valuespace should override this method to hide + hidden parts from the returned items. For example, adding either + a hidden key space suffix or hidden val space proem to ensure insertion + order. Use getFullItemIter instead to return full items with hidden parts + shown for debugging or testing. + + Returns: + items (Iterator[tuple[key,val]]): (key, val) tuples of each item + over the all the items in subdb whose key startswith key made from + keys. Keys may be keyspace prefix to return branches of key space. + When keys is empty then returns all items in subdb + + Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str|bytes|memoryview|Iterable[str|bytes|memoryview]): of key + parts that may be + a truncation of a full keys tuple in in order to address all the + items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value - Returns: - result (bool): True if key exists so delete successful. False otherwise """ - return(self.db.delTopVal(db=self.sdb, key=self._tokey(keys))) + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): + yield (self._tokeys(key), self._des(val)) def cntAll(self): """ @@ -202,15 +329,27 @@ def cntAll(self): """ return self.db.cnt(db=self.sdb) + class Suber(SuberBase): """ - Sub DB of LMDBer. Subclass of SuberBase + Subclass of SuberBase with no LMDB duplicates (i.e. multiple values at same key). """ def __init__(self, db: dbing.LMDBer, *, subkey: str = 'docs.', dupsort: bool=False, **kwa): """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key @@ -286,89 +425,234 @@ def rem(self, keys: Union[str, Iterable]): return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) - -class CesrSuberBase(SuberBase): +class OnSuberBase(SuberBase): """ - Sub class of Suber where data is CESR encode/decode ducktyped subclass - instance such as Matter, Indexer, Counter with .qb64b property when provided - as fully qualified serialization - Automatically serializes and deserializes from qb64b to/from CESR instances + Subclass of SuberBase that adds methods for keys with exposed key part suffix + that is 32 byte serializaton of monotonically increasing ordinal number on + such as sn or fn. + Each key consistes of top key joined with .sep to ordinal suffix + Works with dupsort==True or False """ - def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): + def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key - klas (Type[coring.Matter]): Class reference to subclass of Matter or - Indexer or Counter or any ducktyped class of Matter + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Default False + sep (str): separator to convert keys iterator to key bytes for db key + Default '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ - super(CesrSuberBase, self).__init__(*pa, **kwa) - self.klas = klas + super(OnSuberBase, self).__init__(*pa, **kwa) - def _ser(self, val: coring.Matter): + def putOn(self, keys: str | bytes | memoryview, on: int=0, + val: str | bytes | memoryview=''): """ - Serialize value to bytes to store in db + Returns + result (bool): True if onkey made from key+sep+serialized on is + not found in database so value is written + idempotently. + False otherwise + Parameters: - val (coring.Matter): instance Matter ducktype with .qb64b attribute + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + val (str | bytes | memoryview): serialization """ - return val.qb64b + return (self.db.putOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + def pinOn(self, keys: str | bytes | memoryview, on: int=0, + val: str | bytes | memoryview=''): + """ + Returns + result (bool): True if value is written or overwritten at onkey + False otherwise - def _des(self, val: Union[str, memoryview, bytes]): + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + val (str | bytes | memoryview): serialization """ - Deserialize val to str + return (self.db.setOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + Parameters: - val (Union[str, memoryview, bytes]): convertable to coring.matter + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization """ - if isinstance(val, memoryview): # memoryview is always bytes - val = bytes(val) # convert to bytes - return self.klas(qb64b=val) # converts to bytes + return (self.db.appendOnVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) -class CesrSuber(CesrSuberBase, Suber): + def getOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + val (str): serialization at onkey if any + None if no entry at onkey + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + """ + val = self.db.getOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode()) + return (self._des(val) if val is not None else None) + + + + def remOn(self, keys: str | bytes | memoryview, on: int=0): + """ + Returns + result (bool): True if onkey made from key+sep+serialized on is + found in database so value is removed + Removes all duplicates if any at onkey. + False otherwise + + Parameters: + keys (str | bytes | memoryview | Iterable): keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(key ,on) to form key. + """ + return (self.db.delOnVal(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())) + + + def cntOn(self, keys: str | bytes | memoryview = "", on: int=0): + """ + Returns + cnt (int): count of of all ordinal suffix keyed vals with same + key prefix but different on in onkey in db starting at ordinal + number on where key is formed with onKey(key,on). Count at + each onkey includes duplicates if any. + + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form top key + When keys is empty then counts whole database including + duplicates if any. + on (int): ordinal number used with onKey(key,on) to form key. + """ + return (self.db.cntOnVals(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())) + + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[bytes]): of val with same key but increments of + on >= on i.e. all key.on beginning with on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key + When keys is empty then retrieves whole database including + duplicates if any + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval + sep (bytes): separator character for split + """ + for val in (self.db.getOnValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(key, on, val)]): triples of key, on, val with same + key but increments of on >= on i.e. all key.on beginning with on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form actual key + When keys is empty then retrieves whole database including + duplicates if any + on (int): ordinal number used with onKey(pre,on) to form key at at + which to initiate retrieval + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + +class OnSuber(OnSuberBase, Suber): """ - Sub class of Suber where data is CESR encode/decode ducktyped subclass - instance such as Matter, Indexer, Counter with .qb64b property when provided - as fully qualified serialization. - Extents Suber to support val that are ducktyped CESR serializable .qb64 .qb64b - subclasses such as coring.Matter, coring.Indexer, coring.Counter. - Automatically serializes and deserializes from qb64b to/from CESR instances + Subclass of OnSuberBase andSuber that adds methods for keys with ordinal + numbered suffixes. + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key - klas (Type[coring.Matter]): Class reference to subclass of Matter or - Indexer or Counter or any ducktyped class of Matter + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ - super(CesrSuber, self).__init__(*pa, **kwa) + super(OnSuber, self).__init__(*pa, **kwa) -class CatCesrSuberBase(CesrSuberBase): +class B64SuberBase(SuberBase): """ - Base Class whose values stored in db are a concatenation of the .qb64b property - from one or more subclass instances (qb64b is bytes of fully qualified - serialization) that support CESR encode/decode ducktyped subclass instance - such as Matter, Indexer, Counter - Automatically serializes and deserializes from qb64b to/from CESR instances + Base Class whose values are Iterables of Base64 str or bytes that are stored + in db as .sep joined Base64 bytes. Separator character must not be valid + Base64 character so the split will work unambiguously. + + Automatically joins and splits along separator to Iterable (tuple) of Base64 Attributes: db (dbing.LMDBer): base LMDB db sdb (lmdb._Database): instance of lmdb named sub db for this Suber sep (str): separator for combining keys tuple of strs into key bytes - klas (Iterable): of Class references to subclasses of CESR compatible - , each of to Type[coring.Matter etc] """ - def __init__(self, *pa, klas: Iterable = None, **kwa): + def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -376,36 +660,94 @@ def __init__(self, *pa, klas: Iterable = None, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ - if klas is None: - klas = (coring.Matter, ) # set default to tuple of single Matter - if not nonStringIterable(klas): # not iterable - klas = (klas, ) # make it so - super(CatCesrSuberBase, self).__init__(*pa, klas=klas, **kwa) - # self.klas = klas + super(B64SuberBase, self).__init__(*pa, **kwa) + if Reb64.match(self.sep.encode()): + raise ValueError("Invalid sep={self.sep}, must not be Base64 char.") - def _ser(self, val: Union[Iterable, coring.Matter]): + def _toval(self, vals: str|bytes|memoryview|Iterable[str|bytes|memoryview]): + """ + Converts vals to val bytes with proper separators and returns val bytes. + If vals is already str or bytes or memoryview then returns val bytes. + Else If vals is iterable (non-str) of strs or bytes or memoryview then + joins with .sep and converts to val bytes and returns. + + Returns: + val (bytes): each element of vals is joined by .sep. + + Parameters: + vals (str | bytes | memoryview | Iterable[str | bytes]): db val or + Iterable of (str | bytes | memoryview) to form val. + Note, join of bytes sep works with memoryview. + + """ + if hasattr(vals, "encode"): # str + val = vals.encode("utf-8") + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + if isinstance(vals, memoryview): # memoryview of bytes + val = bytes(vals) # return bytes + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + elif hasattr(vals, "decode"): # bytes + val = vals + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return val + vals = tuple(v.encode() if hasattr(v, "encode") else v for v in vals) # make bytes + for val in vals: + if not (Reb64.match(val)): + raise ValueError(f"Non Base64 {val=}.") + return (self.sep.encode().join(vals)) + + #return (self.sep.join(val.decode() if hasattr(val, "decode") else val + #for val in vals).encode("utf-8")) + + + def _tovals(self, val: bytes | memoryview): + """ + Converts val bytes to vals tuple of strs by decoding and then splitting + at separator .sep. + + Returns: + vals (tuple[str]): makes tuple by splitting val at .sep + + Parameters: + val (bytes | memoryview): db Base64 val. + + """ + if isinstance(val, memoryview): # memoryview of bytes + val = bytes(val) + if hasattr(val, "decode"): # bytes + val = val.decode("utf-8") # convert to str + return tuple(val.split(self.sep)) + + + def _ser(self, val: Union[Iterable, str, bytes]): """ Serialize val to bytes to store in db - Concatenates .qb64b of each instance in objs and returns val bytes + When val is Iterable then joins each elements with .sep returns val bytes Returns: - val (bytes): concatenation of .qb64b of each object instance in vals + val (bytes): .sep join of each Base64 bytes in val Parameters: - subs (Union[Iterable, coring.Matter]): of subclass instances. + val (Union[Iterable, bytes]): of Base64 bytes """ if not nonStringIterable(val): # not iterable val = (val, ) # make iterable - return (b''.join(obj.qb64b for obj in val)) + return (self._toval(val)) - def _des(self, val: Union[str, memoryview, bytes]): + def _des(self, val: memoryview | bytes): """ Converts val bytes to vals tuple of subclass instances by deserializing .qb64b concatenation in order of each instance in .klas @@ -417,72 +759,52 @@ def _des(self, val: Union[str, memoryview, bytes]): val (Union[bytes, memoryview]): of concatenation of .qb64b """ - if not isinstance(val, bytearray): # is memoryview or bytes - val = bytearray(val) # convert so may strip - return tuple(klas(qb64b=val, strip=True) for klas in self.klas) + return self._tovals(val) -class CatCesrSuber(CatCesrSuberBase, Suber): +class B64Suber(B64SuberBase, Suber): """ - Class whose values stored in db are a concatenation of the .qb64b property - from one or more subclass instances (qb64b is bytes of fully qualified - serialization) that support CESR encode/decode ducktyped subclass instance - such as Matter, Indexer, Counter - Automatically serializes and deserializes from qb64b to/from CESR instances + Subclass of B64SuberBase and Suber that serializes and deserializes values + as .sep joined strings of Base64 components. + + .sep must not be Base64 character. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False - Attributes: - db (dbing.LMDBer): base LMDB db - sdb (lmdb._Database): instance of lmdb named sub db for this Suber - sep (str): separator for combining keys tuple of strs into key bytes - klas (Iterable): of Class references to subclasses of CESR compatible - , each of to Type[coring.Matter etc] """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key False (default) means do not enable duplicates at - each key + each key. Set to False sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] - + Must not be Base64 character. + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False """ - super(CatCesrSuber, self).__init__(*pa, **kwa) + super(B64Suber, self).__init__(*pa, **kwa) -class IoSetSuber(SuberBase): - """ - Insertion Ordered Set Suber factory class that supports - a set of distinct entries at a given effective database key but with - dupsort==False. Effective data model is that there are multiple values in a - set of values where every member of the set has the same key (duplicate key). - The set of values is an ordered set using insertion order. Any given value - may appear only once in the set (not a list). - This works similarly to the IO value duplicates for the LMDBer class with a - sub db of LMDB (dupsort==True) but without its size limitation of 511 bytes - for each value when dupsort==True. - Here the key is augmented with a hidden numbered suffix that provides a - an ordered set of values at each effective key (duplicate key). The suffix - is appended and stripped transparently. The set of multiple items with - duplicate keys are retrieved in insertion order when iterating or as a list - of the set elements. +class CesrSuberBase(SuberBase): + """ + Sub class of SuberBase where data is CESR encode/decode ducktyped subclass + instance such as Matter, Indexer, Counter with .qb64b property when provided + as fully qualified serialization + Automatically serializes and deserializes from qb64b to/from CESR instance - Attributes: - db (dbing.LMDBer): base LMDB db - sdb (lmdb._Database): instance of lmdb named sub db for this Suber - sep (str): separator for combining keys tuple of strs into key bytes """ - def __init__(self, db: dbing.LMDBer, *, - subkey: str='docs.', - dupsort: bool=False, **kwa): + + def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -490,41 +812,283 @@ def __init__(self, db: dbing.LMDBer, *, each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + Parameters: + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter + """ - super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa) + super(CesrSuberBase, self).__init__(*pa, **kwa) + self.klas = klas - def put(self, keys: Union[str, Iterable], vals: Iterable): + def _ser(self, val: coring.Matter): """ - Puts all vals at effective key made from keys and hidden ordinal suffix. + Serialize value to bytes to store in db + Parameters: + val (coring.Matter): instance Matter ducktype with .qb64b attribute + """ + return val.qb64b + + + def _des(self, val: memoryview | bytes): + """ + Deserialize val to str + Parameters: + val (memoryview | bytes): convertable to coring.matter + """ + if isinstance(val, memoryview): # memoryview is always bytes + val = bytes(val) # convert to bytes + return self.klas(qb64b=val) # qb64b parameter accepts str + + +class CesrSuber(CesrSuberBase, Suber): + """ + Sub class of Suber where data is CESR encode/decode ducktyped subclass + instance such as Matter, Indexer, Counter with .qb64b property when provided + as fully qualified serialization. + Extends Suber to support val that are ducktyped CESR serializable .qb64 .qb64b + subclasses such as coring.Matter, coring.Indexer, coring.Counter. + Automatically serializes and deserializes from qb64b to/from CESR instances + + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter + """ + super(CesrSuber, self).__init__(*pa, **kwa) + + +class CesrOnSuber(CesrSuberBase, OnSuberBase, Suber): + """ + Subclass of CesrSuberBase, OnSuberBase, and Suber that adds methods for + keys with ordinal numbered suffixes and values that are Cesr serializations + of Matter subclass ducktypes. + + Each key consistes of pre joined with .sep to ordinal suffix + + Assumes dupsort==False + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key. Set to False + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + """ + super(CesrOnSuber, self).__init__(*pa, **kwa) + + +class CatCesrSuberBase(CesrSuberBase): + """ + Base Class whose values stored in db are a concatenation of the .qb64b property + from one or more subclass instances (qb64b is bytes of fully qualified + serialization) that support CESR encode/decode ducktyped subclass instance + such as Matter, Indexer, Counter + Automatically serializes and deserializes from qb64b to/from CESR instances + + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + klas (Iterable): of Class references to subclasses of CESR compatible + , each of to Type[coring.Matter etc] + """ + + def __init__(self, *pa, klas: Iterable = None, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter + + """ + if klas is None: + klas = (coring.Matter, ) # set default to tuple of single Matter + if not nonStringIterable(klas): # not iterable + klas = (klas, ) # make it so + super(CatCesrSuberBase, self).__init__(*pa, klas=klas, **kwa) + + + def _ser(self, val: Union[Iterable, coring.Matter]): + """ + Serialize val to bytes to store in db + Concatenates .qb64b of each instance in val and returns val bytes + + Returns: + cat (bytes): concatenation of .qb64b of each object instance in vals + + Parameters: + val (Union[Iterable, coring.Matter]): of subclass instances. + + """ + if not nonStringIterable(val): # not iterable + val = (val, ) # make iterable + return (b''.join(obj.qb64b for obj in val)) + + + def _des(self, val: memoryview | bytes | bytearray): + """ + Converts val bytes to vals tuple of subclass instances by deserializing + .qb64b concatenation in order of each instance in .klas + + Returns: + vals (tuple): subclass instances + + Parameters: + val (Union[bytes, memoryview]): of concatenation of .qb64b + + """ + if not isinstance(val, bytearray): # is memoryview or bytes + val = bytearray(val) # convert so may strip + return tuple(klas(qb64b=val, strip=True) for klas in self.klas) + + +class CatCesrSuber(CatCesrSuberBase, Suber): + """ + Class whose values stored in db are a concatenation of the .qb64b property + from one or more subclass instances (qb64b is bytes of fully qualified + serialization) that support CESR encode/decode ducktyped subclass instance + such as Matter, Indexer, Counter + Automatically serializes and deserializes from qb64b to/from CESR instances + + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + klas (Iterable): of Class references to subclasses of CESR compatible + , each of to Type[coring.Matter etc] + """ + + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter + + """ + super(CatCesrSuber, self).__init__(*pa, **kwa) + + +class IoSetSuber(SuberBase): + """ + Insertion Ordered Set Suber factory class that supports + a set of distinct entries at a given effective database key but with + dupsort==False. Effective data model is that there are multiple values in a + set of values where every member of the set has the same key (duplicate key). + The set of values is an ordered set using insertion order. Any given value + may appear only once in the set (not a list). + + This works similarly to the IO value duplicates for the LMDBer class with a + sub db of LMDB (dupsort==True) but without its size limitation of 511 bytes + for each value when dupsort==True. + Here the key is augmented with a hidden numbered suffix that provides a + an ordered set of values at each effective key (duplicate key). The suffix + is appended and stripped transparently. The set of multiple items with + duplicate keys are retrieved in insertion order when iterating or as a list + of the set elements. + + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + """ + def __init__(self, db: dbing.LMDBer, *, + subkey: str='docs.', + dupsort: bool=False, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter + """ + super(IoSetSuber, self).__init__(db=db, subkey=subkey, dupsort=False, **kwa) + + + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): + """ + Puts all vals at effective key made from keys and hidden ordinal suffix. that are not already in set of vals at key. Does not overwrite. Parameters: - keys (Iterable): of key strs to be combined in order to form key - vals (Iterable): of str serializations + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): of str serializations Returns: result (bool): True If successful, False otherwise. """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putIoSetVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): """ - Add val to vals at effective key made from keys and hidden ordinal suffix. - that is not already in set of vals at key. Does not overwrite. + Add val idempotently to vals at effective key made from keys and hidden + ordinal suffix. Idempotent means that added value is not already in set + of vals at key. Does not overwrite or add same value at same key more + than once. Parameters: - keys (Iterable): of key strs to be combined in order to form key - val (Union[bytes, str]): serialization + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str | bytes | memoryview): serialization Returns: - result (bool): True means unique value among duplications, - False means duplicte of same value already exists. + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. """ return (self.db.addIoSetVal(db=self.sdb, @@ -533,7 +1097,8 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str, memoryview]): sep=self.sep)) - def pin(self, keys: Union[str, Iterable], vals: Iterable): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at effective key made from keys and hidden ordinal suffix. Overwrites. Removes all pre-existing vals that share same effective keys @@ -547,15 +1112,15 @@ def pin(self, keys: Union[str, Iterable], vals: Iterable): result (bool): True If successful, False otherwise. """ - key = self._tokey(keys) - self.db.delIoSetVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.setIoSetVals(db=self.sdb, - key=key, + key=self._tokey(keys), vals=[self._ser(val) for val in vals], sep=self.sep)) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets vals set list at key made from effective keys @@ -573,24 +1138,7 @@ def get(self, keys: Union[str, Iterable]): sep=self.sep)]) - def getLast(self, keys: Union[str, Iterable]): - """ - Gets last val inserted at effecive key made from keys and hidden ordinal - suffix. - - Parameters: - keys (Iterable): of key strs to be combined in order to form key - - Returns: - val (str): value str, None if no entry at keys - - """ - val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) - return (self._des(val) if val is not None else val) - - - - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets vals iterator at effecive key made from keys and hidden ordinal suffix. All vals in set of vals that share same effecive key are retrieved in @@ -609,32 +1157,37 @@ def getIter(self, keys: Union[str, Iterable]): yield self._des(val) - - def cnt(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ - Return count of values at effective key made from keys and hidden ordinal - suffix. Zero otherwise + Gets last val inserted at effecive key made from keys and hidden ordinal + suffix. Parameters: keys (Iterable): of key strs to be combined in order to form key + + Returns: + val (str): value str, None if no entry at keys + """ - return (self.db.cntIoSetVals(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)) + val = self.db.getIoSetValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) - def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b''): + + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at effective key made from keys and hidden ordinal suffix that matches val if any. Otherwise deletes all values at effective key. Parameters: keys (Iterable): of key strs to be combined in order to form key - val (str): value at key to delete - if val is empty then remove all values at key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key Returns: - result (bool): True if effective key with val exists so delete successful. + result (bool): True if effective key with val exists so rem successful. False otherwise """ @@ -649,117 +1202,62 @@ def rem(self, keys: Union[str, Iterable], val: Union[str, bytes, memoryview]=b'' sep=self.sep) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ - Return iterator over all the items in top branch defined by keys where - keys may be truncation of full branch. - - Returns: - items (Iterator): of (key, val) tuples over the all the items in - subdb whose effective key startswith key made from keys. - Keys may be keyspace prefix in order to return branches of key space. - When keys is empty then returns all items in subdb. - Returned key in each item has ordinal suffix removed. - - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - key, ion = dbing.unsuffix(iokey, sep=self.sep) - yield (self._tokeys(key), self._des(val)) - - - def getIoSetItem(self, keys: Union[str, Iterable]): - """ - Gets (iokeys, val) ioitems list at key made from keys where key is - apparent effective key and ioitems all have same apparent effective key + Return count of values at effective key made from keys and hidden ordinal + suffix. Zero otherwise Parameters: keys (Iterable): of key strs to be combined in order to form key - - Returns: - ioitems (Iterable): each item in list is tuple (iokeys, val) where each - iokeys is actual key tuple including hidden suffix and - each val is str - empty list if no entry at keys - - """ - return ([(self._tokeys(iokey), self._des(val)) for iokey, val in - self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - sep=self.sep)]) - - - def getIoSetItemIter(self, keys: Union[str, Iterable]): """ - Gets (iokeys, val) ioitems iterator at key made from keys where key is - apparent effective key and items all have same apparent effective key - - Parameters: - keys (Iterable): of key strs to be combined in order to form key + return (self.db.cntIoSetVals(db=self.sdb, + key=self._tokey(keys), + sep=self.sep)) - Returns: - ioitems (Iterator): each item iterated is tuple (iokeys, val) where - each iokeys is actual keys tuple including hidden suffix and - each val is str - empty list if no entry at keys. - Raises StopIteration when done + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", + *, topive=False): """ - for iokey, val in self.db.getIoSetItemsIter(db=self.sdb, - key=self._tokey(keys), - sep=self.sep): - yield (self._tokeys(iokey), self._des(val)) - + Return iterator over all the items in top branch defined by keys where + keys may be truncation of full branch. - def getIoItemIter(self, keys: Union[str, Iterable]=b""): - """ Returns: - items (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith effective key made from keys. - Keys may be keyspace prefix to return branches of key space. + items (Iterator): of (key, val) tuples over the all the items in + subdb whose effective key startswith key made from keys. + Keys may be keyspace prefix in order to return branches of key space. When keys is empty then returns all items in subdb. - + Returned key in each item has ordinal suffix removed. Parameters: keys (Iterable): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from + a full keys tuple in in order to address all the items from multiple branches of the key space. If keys is empty then gets - all items in database. Append "" to end of keys Iterable to - ensure get properly separated top branch key. - - """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(iokey), self._des(val)) - - - def remIokey(self, iokeys: Union[str, bytes, memoryview, Iterable]): - """ - Removes entry at keys - - Parameters: - iokeys (Iterable): of key str or tuple of key strs to be combined - in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise - - """ - return self.db.delIoSetIokey(db=self.sdb, iokey=self._tokey(iokeys)) + all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + for key, val in self.db.getTopIoSetItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive), sep=self.sep.encode()): + yield (self._tokeys(key), self._des(val)) class CesrIoSetSuber(CesrSuberBase, IoSetSuber): """ Subclass of CesrSuber and IoSetSuber. - Class whose values stored in db are a concatenation of the .qb64b property - from one or more subclass instances (qb64b is bytes of fully qualified - serialization) that support CESR encode/decode ducktyped subclass instance - such as Matter, Indexer, Counter + Sub class of Suber where data is CESR encode/decode ducktyped subclass + instance such as Matter, Indexer, Counter with .qb64b property when provided + as fully qualified serialization Automatically serializes and deserializes from qb64b to/from CESR instances Extends IoSetSuber with mixin methods ._ser and ._des from CesrSuberBase @@ -787,7 +1285,7 @@ class CesrIoSetSuber(CesrSuberBase, IoSetSuber): def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key @@ -795,8 +1293,10 @@ def __init__(self, *pa, **kwa): each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CesrIoSetSuber, self).__init__(*pa, **kwa) @@ -837,16 +1337,18 @@ class CatCesrIoSetSuber(CatCesrSuberBase, IoSetSuber): """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key dupsort (bool): True means enable duplicates at each key - False (default) means do not enable duplicates at - each key + False (default) means do not enable duplicates at + each key sep (str): separator to convert keys iterator to key bytes for db key default is self.Sep == '.' - klas (Iterable): of Class references to subclasses of Matter, each - of to Type[coring.Matter] + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[coring.Matter]): Class reference to subclass of Matter or + Indexer or Counter or any ducktyped class of Matter """ super(CatCesrIoSetSuber, self).__init__(*pa, **kwa) @@ -865,16 +1367,16 @@ class SignerSuber(CesrSuber): Signer instance to have its .transferable property set correctly. """ - def __init__(self, *pa, klas: Type[coring.Signer] = coring.Signer, **kwa): + def __init__(self, *pa, klas: Type[core.Signer] = core.Signer, **kwa): """ Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key klas (Type[coring.Matter]): Class reference to subclass of Matter """ - if not (issubclass(klas, coring.Signer)): + if not (issubclass(klas, core.Signer)): raise ValueError("Invalid klas type={}, expected {}." - "".format(klas, coring.Signer)) + "".format(klas, core.Signer)) super(SignerSuber, self).__init__(*pa, **kwa) self.klas = klas @@ -907,7 +1409,8 @@ def get(self, keys: Union[str, Iterable]): if val is not None else None) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", + *, topive=False): """ Returns: iterator (Iteratore: tuple (key, val) over the all the items in @@ -916,13 +1419,21 @@ def getItemIter(self, keys: Union[str, Iterable]=b""): returns all items in subdb Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split yield (ikeys, self.klas(qb64b=bytes(val), @@ -943,7 +1454,7 @@ class CryptSignerSuber(SignerSuber): """ def put(self, keys: Union[str, Iterable], val: coring.Matter, - encrypter: coring.Encrypter = None): + encrypter: core.Encrypter = None): """ Puts qb64 of Matter instance val at key made from keys. Does not overwrite If encrypter provided then encrypts first @@ -951,21 +1462,21 @@ def put(self, keys: Union[str, Iterable], val: coring.Matter, Parameters: keys (tuple): of key strs to be combined in order to form key val (Signer): instance of self.klas - encrypter (coring.Encrypter): optional + encrypter (core.Encrypter): optional Returns: result (bool): True If successful, False otherwise, such as key already in database. """ if encrypter: - val = encrypter.encrypt(matter=val) # returns Cipher instance + val = encrypter.encrypt(prim=val) # returns Cipher instance return (self.db.putVal(db=self.sdb, key=self._tokey(keys), val=val.qb64b)) def pin(self, keys: Union[str, Iterable], val: coring.Matter, - encrypter: coring.Encrypter = None): + encrypter: core.Encrypter = None): """ Pins (sets) qb64 of Matter instance val at key made from keys. Overwrites. If encrypter provided then encrypts first @@ -973,20 +1484,20 @@ def pin(self, keys: Union[str, Iterable], val: coring.Matter, Parameters: keys (tuple): of key strs to be combined in order to form key val (Signer): instance of self.klas - encrypter (coring.Encrypter): optional + encrypter (core.Encrypter): optional Returns: result (bool): True If successful. False otherwise. """ if encrypter: - val = encrypter.encrypt(matter=val) # returns Cipher instance + val = encrypter.encrypt(prim=val) # returns Cipher instance return (self.db.setVal(db=self.sdb, key=self._tokey(keys), val=val.qb64b)) - def get(self, keys: Union[str, Iterable], decrypter: coring.Decrypter = None): + def get(self, keys: Union[str, Iterable], decrypter: core.Decrypter = None): """ Gets Signer instance at keys. If decrypter then assumes value in db was encrypted and so decrypts value in db before converting to Signer. @@ -1000,7 +1511,7 @@ def get(self, keys: Union[str, Iterable], decrypter: coring.Decrypter = None): keys (Union[str, iterable]): key strs to be combined in order to form key. Last element of keys is verkey used to determin .transferable for Signer - decrypter (coring.Decrypter): optional. If provided assumes value in + decrypter (core.Decrypter): optional. If provided assumes value in db was encrypted and so decrypts before converting to Signer. Usage: @@ -1017,48 +1528,54 @@ def get(self, keys: Union[str, Iterable], decrypter: coring.Decrypter = None): keys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=keys[-1]) # last split if decrypter: - return (decrypter.decrypt(ser=bytes(val), + return (decrypter.decrypt(qb64=bytes(val), transferable=verfer.transferable)) return (self.klas(qb64b=bytes(val), transferable=verfer.transferable)) - def getItemIter(self, keys: Union[str, Iterable]=b"", - decrypter: coring.Decrypter = None): + def getItemIter(self, keys: str|bytes|memoryview|Iterable= "", + decrypter: core.Decrypter = None, *, topive=False): """ Returns: - iterator (Iterator): of tuples (key, val) over the all the items in + items (Iterator): of tuples (key, val) over the all the items in subdb whose key startswith key made from keys. Keys may be keyspace prefix to return branches of key space. When keys is empty then returns all items in subdb - decrypter (coring.Decrypter): optional. If provided assumes value in + decrypter (core.Decrypter): optional. If provided assumes value in db was encrypted and so decrypts before converting to Signer. Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of + keys (Iterable): tuple of bytes or strs that may be a truncation of a full keys tuple in in order to get all the items from multiple branches of the key space. If keys is empty then gets all items in database. + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): + for key, val in self.db.getTopItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): ikeys = self._tokeys(key) # verkey is last split if any verfer = coring.Verfer(qb64b=ikeys[-1]) # last split if decrypter: - yield (ikeys, decrypter.decrypt(ser=bytes(val), + yield (ikeys, decrypter.decrypt(qb64=bytes(val), transferable=verfer.transferable)) else: yield (ikeys, self.klas(qb64b=bytes(val), transferable=verfer.transferable)) - -class SerderSuber(Suber): +class SerderSuberBase(SuberBase): """ - Sub class of Suber where data is serialized Serder Subclass instance + Sub class of SuberBase where data is serialized Serder Subclass instance given by .klas Automatically serializes and deserializes using .klas Serder methods - """ def __init__(self, *pa, @@ -1068,197 +1585,153 @@ def __init__(self, *pa, Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False - Parameters: + Overridden Parameters: klas (Type[serdering.Serder]): Class reference to subclass of Serder """ - super(SerderSuber, self).__init__(*pa, **kwa) + super(SerderSuberBase, self).__init__(*pa, **kwa) self.klas = klas - def put(self, keys: Union[str, Iterable], val: serdering.SerderKERI): - """ - Puts val at key made from keys. Does not overwrite - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. - """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - - def pin(self, keys: Union[str, Iterable], val: serdering.SerderKERI): - """ - Pins (sets) val at key made from keys. Overwrites. - - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Serder): instance - - Returns: - result (bool): True If successful. False otherwise. + def _ser(self, val: serdering.Serder): """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) - - - def get(self, keys: Union[str, Iterable]): - """ - Gets Serder at keys - - Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - Serder: - None: if no entry at keys - - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here - - """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return self.klas(raw=bytes(val)) if val is not None else None - - - def rem(self, keys: Union[str, Iterable]): - """ - Removes entry at keys - + Serialize value to bytes to store in db Parameters: - keys (tuple): of key strs to be combined in order to form key - - Returns: - result (bool): True if key exists so delete successful. False otherwise + val (serdering.Serder): instance Serder subclass like SerderKERI """ - return(self.db.delVal(db=self.sdb, key=self._tokey(keys))) + return val.raw - def getItemIter(self, keys: Union[str, Iterable]=b""): + def _des(self, val: (str | memoryview | bytes)): """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb - + Deserialize val to str Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. - + val (Union[str, memoryview, bytes]): convertable to coring.matter """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield self._tokeys(iokey), self.klas(raw=bytes(val)) + if isinstance(val, memoryview): # memoryview is always bytes + val = bytes(val) # convert to bytes + elif hasattr(val, "encode"): # str + val = val.encode() # convert to bytes + return self.klas(raw=val, verify=self.verify) -class SchemerSuber(Suber): +class SerderSuber(SerderSuberBase, Suber): """ - Sub class of Suber where data is serialized Schemer instance - Automatically serializes and deserializes using Schemer methods - + Sub class of SerderSuberBase, Suber where data is serialized Serder Subclass + instance given by .klas + Automatically serializes and deserializes using .klas Serder methods """ def __init__(self, *pa, **kwa): """ - Parameters: + Inherited Parameters: db (dbing.LMDBer): base db subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder """ - super(SchemerSuber, self).__init__(*pa, **kwa) - - def put(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Puts val at key made from keys. Does not overwrite + super(SerderSuber, self).__init__(*pa, **kwa) - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance - Returns: - result (bool): True If successful, False otherwise, such as key - already in database. - """ - return (self.db.putVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) +class SerderIoSetSuber(SerderSuberBase, IoSetSuber): + """ + Sub class of SerderSuberBase and IoSetSuber that allows multiple Serder + instances to be stored at the same db key in insertion order. + Example use case would be an escrow where the key is a sequence number + based index (such as snKey). - def pin(self, keys: Union[str, Iterable], val: scheming.Schemer): - """ - Pins (sets) val at key made from keys. Overwrites. + Sub class of SerderSuberBase where data is serialized Serder Subclass instance + given by .klas + Automatically serializes and deserializes using .klas Serder methods - Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Schemer): instance + Extends IoSetSuber so that all IoSetSuber methods now work with Serder + subclass for each val. - Returns: - result (bool): True If successful. False otherwise. - """ - return (self.db.setVal(db=self.sdb, - key=self._tokey(keys), - val=val.raw)) + IoSetSuber stores at each effective key a set of distinct values that + share that same effective key where each member of the set is retrieved in + insertion order (dupsort==False) + The methods allows an Iterable (set valued) of Iterables of separation subclass + instances to be stored at a given effective key in insertion order. - def get(self, keys: Union[str, Iterable]): - """ - Gets Serder at keys + Actual keys include a hidden ordinal key suffix that tracks insertion order. + The suffix is appended and stripped transparently from the keys. The set of + items with duplicate effective keys are retrieved in insertion order when + iterating or as a list of the set elements. The actual iokey for any item + includes the ordinal suffix. - Parameters: - keys (tuple): of key strs to be combined in order to form key + Attributes: + db (dbing.LMDBer): base LMDB db + sdb (lmdb._Database): instance of lmdb named sub db for this Suber + sep (str): separator for combining keys tuple of strs into key bytes + klas (Iterable): of Class references to subclasses of CESR compatible + , each of to Type[coring.Matter etc] - Returns: - Schemer: - None: if no entry at keys + """ - Usage: - Use walrus operator to catch and raise missing entry - if (srder := mydb.get(keys)) is None: - raise ExceptionHere - use srdr here + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[serdering.Serder]): Class reference to subclass of Serder """ - val = self.db.getVal(db=self.sdb, key=self._tokey(keys)) - return scheming.Schemer(raw=bytes(val)) if val is not None else None + super(SerderIoSetSuber, self).__init__(*pa, **kwa) - def rem(self, keys: Union[str, Iterable]): - """ - Removes entry at keys - Parameters: - keys (tuple): of key strs to be combined in order to form key - Returns: - result (bool): True if key exists so delete successful. False otherwise - """ - return self.db.delVal(db=self.sdb, key=self._tokey(keys)) - def getItemIter(self, keys: Union[str, Iterable]=b""): - """ - Returns: - iterator (Iterator): tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb +class SchemerSuber(SerderSuberBase, Suber): + """ + Sub class of SerderSuberBase and Suber where data is serialized Schemer instance + Schemer ser/des is ducktype of Serder using .raw + Automatically serializes and deserializes using Schemer methods + """ - Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + def __init__(self, *pa, + klas: Type[ scheming.Schemer] = scheming.Schemer, + **kwa): + """ + Inherited Parameters: + db (dbing.LMDBer): base db + subkey (str): LMDB sub database key + dupsort (bool): True means enable duplicates at each key + False (default) means do not enable duplicates at + each key + sep (str): separator to convert keys iterator to key bytes for db key + default is self.Sep == '.' + verify (bool): True means reverify when ._des from db when applicable + False means do not reverify. Default False + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder + Overridden Parameters: + klas (Type[scheming.Schemer]): Class reference to ducktyped subclass + of Serder intercepts passed in klas and forces it to Schemer """ - for iokey, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield self._tokeys(iokey), scheming.Schemer(raw=bytes(val)) + if not issubclass(klas, scheming.Schemer): + raise TypeError(f"Invalid {klas=}, not subclass of {scheming.Schemer}.") + super(SchemerSuber, self).__init__(*pa, klas=klas, **kwa) class DupSuber(SuberBase): @@ -1282,7 +1755,8 @@ def __init__(self, db: Type[dbing.LMDBer], *, super(DupSuber, self).__init__(db=db, subkey=subkey, dupsort=True, **kwa) - def put(self, keys: Union[str, Iterable], vals: list): + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Puts all vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1291,8 +1765,10 @@ def put(self, keys: Union[str, Iterable], vals: list): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes of each value to be written at key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes of each + value to be written at key Returns: result (bool): True If successful, False otherwise. @@ -1300,12 +1776,15 @@ def put(self, keys: Union[str, Iterable], vals: list): Apparently always returns True (how .put works with dupsort=True) """ + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=self._tokey(keys), vals=[self._ser(val) for val in vals])) - def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview ): """ Add val to vals at key made from keys. Does not overwrite. Adds to existing dup values at key if any. Duplicate means another entry at the same key @@ -1314,8 +1793,8 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): unless it is a unique value for that key. Parameters: - keys (tuple): of key strs to be combined in order to form key - val (Union[str, bytes]): value + keys (str | bytes | memoryview | Iterable): of key strs to be combined in order to form key + val (str | bytes | memoryview): value Returns: result (bool): True means unique value among duplications, @@ -1327,14 +1806,16 @@ def add(self, keys: Union[str, Iterable], val: Union[bytes, str]): val=self._ser(val))) - def pin(self, keys: Union[str, Iterable], vals: list): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ Pins (sets) vals at key made from keys. Overwrites. Removes all pre-existing dup vals and replaces them with vals Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): str or bytes values + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + vals (str | bytes | memoryview | Iterable): str or bytes values Returns: result (bool): True If successful, False otherwise. @@ -1342,18 +1823,21 @@ def pin(self, keys: Union[str, Iterable], vals: list): """ key = self._tokey(keys) self.db.delVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable return (self.db.putVals(db=self.sdb, key=key, vals=[self._ser(val) for val in vals])) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals list at key made from keys Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: vals (list): each item in list is str @@ -1364,7 +1848,7 @@ def get(self, keys: Union[str, Iterable]): self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] - def getLast(self, keys: Union[str, Iterable]): + def getLast(self, keys: str | bytes | memoryview | Iterable): """ Gets last dup val at key made from keys @@ -1379,14 +1863,15 @@ def getLast(self, keys: Union[str, Iterable]): return self._des(val) if val is not None else val - def getIter(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ Gets dup vals iterator at key made from keys Duplicates are retrieved in lexocographic order not insertion order. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key Returns: iterator: vals each of str. Raises StopIteration when done @@ -1396,17 +1881,19 @@ def getIter(self, keys: Union[str, Iterable]): yield self._des(val) - def cnt(self, keys: Union[str, Iterable]): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ Return count of dup values at key made from keys, zero otherwise Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key """ return (self.db.cntVals(db=self.sdb, key=self._tokey(keys))) - def rem(self, keys: Union[str, Iterable], val=b''): + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = b''): """ Removes entry at keys @@ -1419,181 +1906,519 @@ def rem(self, keys: Union[str, Iterable], val=b''): result (bool): True if key exists so delete successful. False otherwise """ - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=self._ser(val))) + if val: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) + else: + return (self.db.delVals(db=self.sdb, + key=self._tokey(keys))) -class CesrDupSuber(DupSuber): + +class CesrDupSuber(CesrSuberBase, DupSuber): """ - Sub class of DupSuber that supports multiple entries at each key (duplicates) - with dupsort==True, where data where data is Matter.qb64b property - which is a fully qualified serialization of matter subclass instance - Automatically serializes and deserializes from qb64b to/from Matter instances + Sub class of DupSuber whose values are CESR ducktypes of Matter subclasses. + serialized to and deserialied from val instance .qb64b property + which is a fully qualified serialization. + Automatically serializes and deserializes from qb64b to/from Matter ducktyped + instances + DupSuber supports multiple entries at each key (duplicates) with dupsort==True Do not use if serialized value is greater than 511 bytes. This is a limitation of dupsort==True sub dbs in LMDB """ - def __init__(self, *pa, klas: Type[coring.Matter] = coring.Matter, **kwa): + def __init__(self, *pa, **kwa): """ - Parameters: - db (dbing.LMDBer): base db - subkey (str): LMDB sub database key - klas (Type[coring.Matter]): Class reference to subclass of Matter + Inherited Parameters: + """ super(CesrDupSuber, self).__init__(*pa, **kwa) - self.klas = klas - def put(self, keys: Union[str, Iterable], vals: list): +class IoDupSuber(DupSuber): + """ + Sub class of DupSuber that supports Insertion Ordering (IoDup) of duplicates + By automagically prepending and stripping ordinal proem to/from each + duplicate value at a given key. + + IoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + IoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + + """ + super(IoDupSuber, self).__init__(*pa, **kwa) + + + def put(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ - Puts all vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. + Puts all vals idempotently at key made from keys in insertion order using + hidden ordinal proem. Idempotently means do not put any val in vals that is + already in dup vals at key. Does not overwrite. Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): of str serializations Returns: result (bool): True If successful, False otherwise. - Apparently always returns True (how .put works with dupsort=True) - """ - return (self.db.putVals(db=self.sdb, - key=self._tokey(keys), - vals=[val.qb64b for val in vals])) + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return (self.db.putIoDupVals(db=self.sdb, + key=self._tokey(keys), + vals=[self._ser(val) for val in vals])) - def add(self, keys: Union[str, Iterable], val: coring.Matter): + def add(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview): """ - Add val to vals at key made from keys. Does not overwrite. Adds to existing - dup values at key if any. Duplicate means another entry at the same key - but the entry is still a unique value. Duplicates are inserted in - lexocographic order not insertion order. Lmdb does not insert a duplicate - unless it is a unique value for that key. + Add val idempotently at key made from keys in insertion order using hidden + ordinal proem. Idempotently means do not add val that is already in + dup vals at key. Does not overwrite. Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance (subclass) + keys (Iterable): of key strs to be combined in order to form key + val (str | bytes | memoryview): serialization Returns: - result (bool): True means unique value among duplications, - False means duplicte of same value already exists. + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. """ - return (self.db.addVal(db=self.sdb, - key=self._tokey(keys), - val=val.qb64b)) + return (self.db.addIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val))) - def pin(self, keys: Union[str, Iterable], vals: list): + def pin(self, keys: str | bytes | memoryview | Iterable, + vals: str | bytes | memoryview | Iterable): """ - Pins (sets) vals at key made from keys. Overwrites. Removes all - pre-existing dup vals and replaces them with vals + Pins (sets) vals at key made from keys in insertion order using hidden + ordinal proem. Overwrites. Removes all pre-existing vals that share + same keys and replaces them with vals Parameters: - keys (tuple): of key strs to be combined in order to form key - vals (list): instances of coring.Matter (subclass) + keys (Iterable): of key strs to be combined in order to form key + vals (Iterable): str serializations Returns: result (bool): True If successful, False otherwise. """ key = self._tokey(keys) - self.db.delVals(db=self.sdb, key=key) # delete all values - return (self.db.putVals(db=self.sdb, - key=key, - vals=[val.qb64b for val in vals])) + self.db.delIoDupVals(db=self.sdb, key=key) # delete all values + if not nonStringIterable(vals): # not iterable + vals = (vals, ) # make iterable + return self.db.putIoDupVals(db=self.sdb, + key=key, + vals=[self._ser(val) for val in vals]) - def get(self, keys: Union[str, Iterable]): + def get(self, keys: str | bytes | memoryview | Iterable): """ - Gets dup vals list at key made from keys + Gets vals dup list in insertion order using key made from keys and + hidden ordinal proem on dups. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (Iterable): of key strs to be combined in order to form key Returns: - vals (list): each item in list is instance of self.klas + vals (Iterable): each item in list is str empty list if no entry at keys """ - return [self.klas(qb64b=bytes(val)) for val in - self.db.getValsIter(db=self.sdb, key=self._tokey(keys))] + return ([self._des(val) for val in + self.db.getIoDupVals(db=self.sdb, key=self._tokey(keys))]) - def getLast(self, keys: Union[str, Iterable]): + def getIter(self, keys: str | bytes | memoryview | Iterable): """ - Gets last dup val at key made from keys + Gets vals dup iterator in insertion order using key made from keys and + hidden ordinal proem on dups. + All vals in dups that share same key are retrieved in insertion order. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key parts Returns: - val (str): instance of self.klas else None if no value at key + vals (Iterator): str values. Raises StopIteration when done """ - val = self.db.getValLast(db=self.sdb, key=self._tokey(keys)) - if val is not None: - val = self.klas(qb64b=bytes(val)) - return val + for val in self.db.getIoDupValsIter(db=self.sdb, + key=self._tokey(keys)): + yield self._des(val) + + + def getLast(self, keys: str | bytes | memoryview | Iterable): + """ + Gets last val inserted at key made from keys in insertion order using + hidden ordinal proem. + Parameters: + keys (Iterable): of key strs to be combined in order to form key + Returns: + val (str): value str, None if no entry at keys - def getIter(self, keys: Union[str, Iterable]): """ - Gets dup vals iterator at key made from keys + val = self.db.getIoDupValLast(db=self.sdb, key=self._tokey(keys)) + return (self._des(val) if val is not None else val) - Duplicates are retrieved in lexocographic order not insertion order. + + def rem(self, keys: str | bytes | memoryview | Iterable, + val: str | bytes | memoryview = ''): + """ + Removes entry at key made from keys and dup val that matches val if any, + notwithstanding hidden ordinal proem. Otherwise deletes all dup values + at key if any. Parameters: - keys (tuple): of key strs to be combined in order to form key + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key Returns: - iterator: vals each of self.klas. Raises StopIteration when done + result (bool): True if key with dup val exists so rem successful. + False otherwise """ - for val in self.db.getValsIter(db=self.sdb, key=self._tokey(keys)): - yield self.klas(qb64b=bytes(val)) + if val: + return self.db.delIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val)) + else: + return self.db.delIoDupVals(db=self.sdb, key=self._tokey(keys)) - def rem(self, keys: Union[str, Iterable], val=None): + def cnt(self, keys: str | bytes | memoryview | Iterable): """ - Removes entry at keys + Return count of dup values at key made from keys with hidden ordinal + proem. Zero otherwise Parameters: - keys (tuple): of key strs to be combined in order to form key - val (coring.Matter): instance of coring.Matter subclass dup val - at key to delete - if val is None then remove all values at key + keys (str | bytes | memoryview | Iterable): of key parts to be + combined in order to form key + """ + return (self.db.cntIoDupVals(db=self.sdb, key=self._tokey(keys))) + + + def getItemIter(self, keys: str | bytes | memoryview | Iterable = "", + *, topive=False): + """ + Return iterator over all the items including dup items for all keys + in top branch defined by keys where keys may be truncation of full branch. Returns: - result (bool): True if key exists so delete successful. False otherwise + items (Iterator): of (key, val) tuples over the all the items in + subdb whose key startswith key made from keys and val has its hidden + dup ordinal proem removed. + Keys may be keyspace prefix in order to return branches of key space. + When keys is empty then returns all items in subdb. + + Parameters: + keys (str | bytes | memoryview | Iterable): key or key parts that + may be a truncation of a full keys tuple in in order to address + all the items from multiple branches of the key space. + If keys is empty then gets all items in database. + Either append "" to end of keys Iterable to ensure get properly + separated top branch key or use top=True. + In Python str.startswith('') always returns True so if branch + key is empty string it matches all keys in db with startswith. + + topive (bool): True means treat as partial key tuple from top branch of + key space given by partial keys. Resultant key ends in .sep + character. + False means treat as full branch in key space. Resultant key + does not end in .sep character. + When last item in keys is empty str then will treat as + partial ending in sep regardless of top value + + """ + + for key, val in self.db.getTopIoDupItemIter(db=self.sdb, + top=self._tokey(keys, topive=topive)): + yield (self._tokeys(key), self._des(val)) + + +class OnIoDupSuber(OnSuberBase, IoDupSuber): + """ + Sub class of IoDupSuber and OnSuberBase that supports Insertion Ordering + (IoDup) of duplicates but where the trailing part of the key space is + a serialized monotonically increasing ordinal number. This is useful for + escrows of key events etc where duplicates of likely events are maintained + in insertion order. + Insertion order is maintained by automagically prepending and stripping an + ordinal ordering proem to/from each duplicate value at a given key. + + OnIoDupSuber adds the convenience methods from OnSuberBase to IoDupSuber for + those cases where the keyspace has a trailing ordinal part. + + There are two ordinals, one in the key space and a hidden one in the duplicate + data value space. + + OnIoDupSuber supports insertion ordered multiple entries at each key + (duplicates) with dupsort==True + + Do not use if serialized length key + proem + value, is greater than 511 bytes. + This is a limitation of dupsort==True sub dbs in LMDB + + OnIoDupSuber may be more performant then IoSetSuber for values that are indices + to other sub dbs that fit the size constraint because LMDB support for + duplicates is more space efficient and code performant. + + Duplicates at a given key preserve insertion order of duplicate. + Because lmdb is lexocographic an insertion ordering proem is prepended to + all values that makes lexocographic order that same as insertion order. + + Duplicates are ordered as a pair of key plus value so prepending proem + to each value changes duplicate ordering. Proem is 33 characters long. + With 32 character hex string followed by '.' for essentiall unlimited + number of values which will be limited by memory. + + With prepended proem ordinal must explicity check for duplicate values + before insertion. Uses a python set for the duplicate inclusion test. + Set inclusion scales with O(1) whereas list inclusion scales with O(n). + """ + def __init__(self, *pa, **kwa): + """ + Inherited Parameters: + + """ + super(OnIoDupSuber, self).__init__(*pa, **kwa) + + + def addOn(self, keys: str | bytes | memoryview | Iterable, on: int=0, + val: str | bytes | memoryview = ''): + """ + Add val idempotently at key made from keys in insertion order using hidden + ordinal proem. Idempotently means do not add val that is already in + dup vals at key. Does not overwrite. + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form onkey + on (int): ordinal number used with onKey(pre,on) to form onkey. + val (str | bytes | memoryview): serialization + + Returns: + result (bool): True means unique value added among duplications, + False means duplicate of same value already exists. + + """ + return (self.db.addOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode())) + + + + def appendOn(self, keys: str | bytes | memoryview, + val: str | bytes | memoryview): + """ + Returns: + on (int): ordinal number of newly appended val + + Parameters: + keys (str | bytes | memoryview | Iterable): top keys as prefix to be + combined with serialized on suffix and sep to form key + val (str | bytes | memoryview): serialization + """ + return (self.db.appendOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + val=self._ser(val), + sep=self.sep.encode())) + + def getOn(self, keys: str | bytes | memoryview | Iterable, on: int = 0): """ - if val is not None: - val = val.qb64b + Gets dup vals list at key made from keys + + Parameters: + keys (str | bytes | memoryview | Iterable): of key strs to be + combined in order to form key + on (int): ordinal number used with onKey(pre,on) to form key. + + Returns: + vals (list): each item in list is str + empty list if no entry at keys + + """ + return [self._des(val) for val in + self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode())] + + + def remOn(self, keys: str | bytes | memoryview | Iterable, on: int=0, + val: str | bytes | memoryview = ''): + """ + Removes entry at key made from keys and dup val that matches val if any, + notwithstanding hidden ordinal proem. Otherwise deletes all dup values + at key if any. + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + + on (int): ordinal number used with onKey(pre,on) to form key. + val (str): value at key to delete. Subclass ._ser method may + accept different value types + if val is empty then remove all values at key + + Returns: + result (bool): True if onkey with dup val exists so rem successful. + False otherwise + + """ + if val: + return self.db.delOnIoDupVal(db=self.sdb, + key=self._tokey(keys), + on=on, + val=self._ser(val), + sep=self.sep.encode()) else: - val = b'' - return (self.db.delVals(db=self.sdb, key=self._tokey(keys), val=val)) + return self.db.delOnIoDupVals(db=self.sdb, + key=self._tokey(keys), + on=on, + sep=self.sep.encode()) + + + + def getOnIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + val (Iterator[bytes]): deserialized val of of each + onkey + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) - def getItemIter(self, keys: Union[str, Iterable]=b""): + def getOnItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): """ Returns: - iterator (Iteratore: tuple (key, val) over the all the items in - subdb whose key startswith key made from keys. Keys may be keyspace - prefix to return branches of key space. When keys is empty then - returns all items in subdb + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, + deserialized val) Parameters: - keys (Iterator): tuple of bytes or strs that may be a truncation of - a full keys tuple in in order to get all the items from - multiple branches of the key space. If keys is empty then gets - all items in database. + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + def getOnLastIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + last (Iterator[bytes]): deserialized last duplicate val of of each + onkey + + Parameters: + keys (str | bytes | memoryview | iterator): top keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupLastValIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnLastItemIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + items (Iterator[(top keys, on, val)]): triples of (keys, on int, + deserialized val) last duplicate item as each onkey where onkey + is the key+serialized on + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form key + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for keys, on, val in (self.db.getOnIoDupLastItemIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) + + + def getOnBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns + val (Iterator[bytes]): deserialized val of of each + onkey in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split + """ + for val in (self.db.getOnIoDupValBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._des(val)) + + + def getOnItemBackIter(self, keys: str|bytes|memoryview|Iterable = "", on: int=0): + """ + Returns: + items (Iterator[(top keys, on, val)]): triples of (onkeys, on int, + deserialized val) in reverse order + + Parameters: + keys (str | bytes | memoryview | iterator): keys as prefix to be + combined with serialized on suffix and sep to form onkey + When keys is empty then retrieves whole database including duplicates + on (int): ordinal number used with onKey(pre,on) to form key. + sep (bytes): separator character for split """ - for key, val in self.db.getTopItemIter(db=self.sdb, key=self._tokey(keys)): - yield (self._tokeys(key), self.klas(qb64b=bytes(val))) + for keys, on, val in (self.db.getOnIoDupItemBackIter(db=self.sdb, + key=self._tokey(keys), on=on, sep=self.sep.encode())): + yield (self._tokeys(keys), on, self._des(val)) diff --git a/src/keri/demo/demo_bob.py b/src/keri/demo/demo_bob.py index 07c7fccc7..aa812b2dd 100644 --- a/src/keri/demo/demo_bob.py +++ b/src/keri/demo/demo_bob.py @@ -10,10 +10,15 @@ import logging from keri import __version__ + +from keri import help + +from keri import core + from keri.app import directing from keri.demo import demoing -from keri.core import coring -from keri import help + + def runDemo(name="bob", remote=5621, local=5620, expire=0.0): @@ -24,7 +29,7 @@ def runDemo(name="bob", remote=5621, local=5620, expire=0.0): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/src/keri/demo/demo_eve.py b/src/keri/demo/demo_eve.py index 688dea03b..895e28dbf 100644 --- a/src/keri/demo/demo_eve.py +++ b/src/keri/demo/demo_eve.py @@ -10,10 +10,14 @@ import logging from keri import __version__ +from keri import help # logger support + +from keri import core + from keri.app import directing from keri.demo import demoing -from keri.core import coring -from keri import help # logger support + + def runDemo(name="eve", remote=5620, local=5621, expire=0.0): """ @@ -24,7 +28,7 @@ def runDemo(name="eve", remote=5620, local=5621, expire=0.0): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/src/keri/demo/demo_kev.py b/src/keri/demo/demo_kev.py index 7b125b0a3..96c70f282 100644 --- a/src/keri/demo/demo_kev.py +++ b/src/keri/demo/demo_kev.py @@ -11,7 +11,8 @@ from keri import help from hio.base import doing -from ..app import apping +from ..app import habbing, keeping, apping +from ..db import basing logger = help.ogler.getLogger() diff --git a/src/keri/demo/demo_sam.py b/src/keri/demo/demo_sam.py index b31d2f053..48ef3411e 100644 --- a/src/keri/demo/demo_sam.py +++ b/src/keri/demo/demo_sam.py @@ -11,10 +11,14 @@ from keri import __version__ +from keri import help # logger support + +from keri import core + from keri.app import directing from keri.demo import demoing -from keri.core import coring -from keri import help # logger support + + def runDemo(name="sam", remote=5621, local=5620, expire=0.0): """ @@ -25,7 +29,7 @@ def runDemo(name="sam", remote=5621, local=5620, expire=0.0): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/src/keri/end/ending.py b/src/keri/end/ending.py index 4746e0bf6..72729316f 100644 --- a/src/keri/end/ending.py +++ b/src/keri/end/ending.py @@ -22,7 +22,7 @@ from .. import help from .. import kering from ..app import habbing -from ..core import coring +from ..core import coring, indexing from ..help import helping logger = help.ogler.getLogger() @@ -82,12 +82,12 @@ def signature(signages): where: markers (Union[list, dict]): When dict each item (key, val) has key as str identifier of marker and has val as instance of - either coring.Siger or coring.Cigar. - When list each item is instance of either coring.Siger or + either indexing.Siger or coring.Cigar. + When list each item is instance of either indexing.Siger or coring.Cigar. All markers must be of same class indexed (bool): True means marker values are indexed signatures - using coring.Siger. False means marker values are unindexed + using indexing.Siger. False means marker values are unindexed signatures using coring.Cigar. None means auto detect from first marker value class. All markers must be of same class. signer (str): optional identifier of signage. May be a @@ -188,12 +188,12 @@ def designature(value): where: markers (Union[list, dict]): When dict each item (key, val) has key as str identifier of marker and has val as instance of - either coring.Siger or coring.Cigar. - When list each item is instance of either coring.Siger or + either indexing.Siger or coring.Cigar. + When list each item is instance of either indexing.Siger or coring.Cigar. All markers must be of same class indexed (bool): True means marker values are indexed signatures - using coring.Siger. False means marker values are unindexed + using indexing.Siger. False means marker values are unindexed signatures using coring.Cigar. None means auto detect from first marker value class. All markers must be of same class. signer (str): optional identifier of signage. May be a @@ -220,7 +220,7 @@ def designature(value): if "indexed" not in items: raise ValueError("Missing indexed field in Signature header signage.") - indexed = items["indexed"] not in kering.FALSEY # make bool + indexed = items["indexed"] not in helping.FALSEY # make bool del items["indexed"] if "signer" in items: @@ -250,7 +250,7 @@ def designature(value): if kind == "CESR": # convert to Siger or Cigar instances for key, val in items.items(): if indexed: - items[key] = coring.Siger(qb64=val) + items[key] = indexing.Siger(qb64=val) else: items[key] = coring.Cigar(qb64=val) @@ -275,7 +275,7 @@ def siginput(name, method, path, headers, fields, hab=None, signers=None, expire expires (str): iso8601 formated date string indicating exiration of header signature signers (list): Optional signer objects used to sign the values hab (Hab): Optional Hab used to sign the values. One of signers or Hab is required - fields (str): Fields in request to sign. Includes special fields as well as Header fields + fields (list): Fields in request to sign. Includes special fields as well as Header fields headers (dict): HTTP request headers path (str): HTTP request path method (str): HTTP request method (POST, GET, PUT, etc) @@ -577,7 +577,7 @@ def on_get(self, req, rep, aid=None, role=None, eid=None): rep.status = falcon.HTTP_NOT_FOUND return - if kever.delegated and kever.delegator not in self.hby.kevers: + if kever.delegated and kever.delpre not in self.hby.kevers: rep.status = falcon.HTTP_NOT_FOUND return diff --git a/src/keri/help/__init__.py b/src/keri/help/__init__.py index 6922651ad..a7bf9ad5a 100644 --- a/src/keri/help/__init__.py +++ b/src/keri/help/__init__.py @@ -6,14 +6,14 @@ utility functions """ +import logging + # Setup module global ogler as package logger factory. This must be done on # import to ensure global is defined so all modules in package have access to # logggers via ogling.ogler.getLoggers(). May always change level and reopen log file # if need be - from hio.help import ogling -import logging # Custom TRACE log level configuration TRACE = 5 # TRACE (5) logging level value between DEBUG (10) and NOTSET (0) @@ -28,4 +28,5 @@ def trace(self, message, *args, **kwargs): # want help.ogler always defined by default ogler = ogling.initOgler(prefix='keri', syslogged=False) # inits once only on first import -from .helping import nowIso8601, toIso8601, fromIso8601 +from .helping import (nowIso8601, toIso8601, fromIso8601, + nonStringSequence, nonStringIterable) diff --git a/src/keri/help/helping.py b/src/keri/help/helping.py index 8599c6751..63bac1a9a 100644 --- a/src/keri/help/helping.py +++ b/src/keri/help/helping.py @@ -7,12 +7,18 @@ import dataclasses import datetime import re +from collections import deque from collections.abc import Iterable, Sequence, Mapping import pysodium -B64REX = b'[A-Za-z-_]*' -B64Rev = re.compile(B64REX) #compile is faster +# usage: +# x not in FALSEY +# x in FALSEY +# x not in TRUTHY +# x in TRUTHY +FALSEY = (False, 0, None, "?0", "no", "false", "False", "off") +TRUTHY = (True, 1, "?1", "yes" "true", "True", 'on') # Utilities @@ -30,7 +36,7 @@ def sceil(r): """ Symmetric ceiling function Returns: - sc (int): value that is symmetric ceiling of r away from zero + sceil (int): value that is symmetric ceiling of r away from zero Because int() provides a symmetric floor towards zero, just inc int(r) by: 1 when r - int(r) > 0 (r positive) @@ -98,62 +104,6 @@ def klasify(sers: Iterable, klases: Iterable, args: Iterable = None): for ser, klas, arg in zip(sers, klases, args)) -def isBase64(sb): - try: - if hasattr(sb, "encode"): - sb = sb.encode("utf-8") - - match = B64Rev.fullmatch(sb) - if match: - return True - return False - except Exception as ex: - return False - - -def keyToKey64u(key): - """ - Returns 64u - Convert and return bytes key to unicode base64 url-file safe version - """ - return base64.urlsafe_b64encode(key).decode("utf-8") - - -def key64uToKey(key64u): - """ - Returns bytes - Convert and return unicode base64 url-file safe key64u to bytes key - """ - return base64.urlsafe_b64decode(key64u.encode("utf-8")) - - -def verifyEd25519(sig, msg, vk): - """ - Returns True if signature sig of message msg is verified with - verification key vk Otherwise False - All of sig, msg, vk are bytes - """ - try: - result = pysodium.crypto_sign_verify_detached(sig, msg, vk) - except Exception as ex: - return False - return (True if result else False) - - -def verify64uEd25519(signature, message, verkey): - """ - Returns True if signature is valid for message with respect to verification - key verkey - - signature and verkey are encoded as unicode base64 url-file strings - and message is unicode string as would be the case for a json object - - """ - sig = key64uToKey(signature) - vk = key64uToKey(verkey) - msg = message.encode("utf-8") - return (verifyEd25519(sig, msg, vk)) - def nonStringIterable(obj): """ @@ -220,7 +170,7 @@ def extractElementValues(element, values): def extractValues(ked, labels): """ Returns list of depth first recursively extracted values from elements of - key event dict ked whose flabels are in lables list + key event dict ked whose labels are in labels list Parameters: ked is key event dict @@ -293,3 +243,169 @@ def fromIso8601(dts): if hasattr(dts, "decode"): dts = dts.decode("utf-8") return (datetime.datetime.fromisoformat(dts)) + + + +# Base64 utilities +BASE64_PAD = b'=' + +# Mappings between Base64 Encode Index and Decode Characters +# B64ChrByIdx is dict where each key is a B64 index and each value is the B64 char +# B64IdxByChr is dict where each key is a B64 char and each value is the B64 index +# Map Base64 index to char +B64ChrByIdx = dict((index, char) for index, char in enumerate([chr(x) for x in range(65, 91)])) +B64ChrByIdx.update([(index + 26, char) for index, char in enumerate([chr(x) for x in range(97, 123)])]) +B64ChrByIdx.update([(index + 52, char) for index, char in enumerate([chr(x) for x in range(48, 58)])]) +B64ChrByIdx[62] = '-' +B64ChrByIdx[63] = '_' +# Map char to Base64 index +B64IdxByChr = {char: index for index, char in B64ChrByIdx.items()} +# tuple +B64_CHARS = tuple(B64ChrByIdx.values()) # tuple of characters in Base64 + + +B64REX = rb'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] bytes +Reb64 = re.compile(B64REX) # compile is faster +# Usage: if Reb64.match(bext): or if not Reb64.match(bext): bext is bytes + + +def intToB64(i, l=1): + """ + Returns conversion of int i to Base64 str + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + d = deque() # deque of characters base64 + + while l: + d.appendleft(B64ChrByIdx[i % 64]) + i = i // 64 + if not i: + break + for j in range(l - len(d)): # range(x) x <= 0 means do not iterate + d.appendleft("A") + return ("".join(d)) + + +def intToB64b(i, l=1): + """ + Returns conversion of int i to Base64 bytes + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + return (intToB64(i=i, l=l).encode("utf-8")) + + +def b64ToInt(s): + """ + Returns conversion of Base64 str s or bytes to int + """ + if not s: + raise ValueError("Empty string, conversion undefined.") + if hasattr(s, 'decode'): + s = s.decode("utf-8") + i = 0 + for e, c in enumerate(reversed(s)): + i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) + return i + + + +def codeB64ToB2(s): + """ + Returns conversion (decode) of Base64 chars to Base2 bytes. + Where the number of total bytes returned is equal to the minimun number of + octets sufficient to hold the total converted concatenated sextets from s, + with one sextet per each Base64 decoded char of s. Assumes no pad chars in s. + Sextets are left aligned with pad bits in last (rightmost) byte. + This is useful for decoding as bytes, code characters from the front of + a Base64 encoded string of characters. + """ + i = b64ToInt(s) + i <<= 2 * (len(s) % 4) # add 2 bits right zero padding for each sextet + n = sceil(len(s) * 3 / 4) # compute min number of ocetets to hold all sextets + return (i.to_bytes(n, 'big')) + + +def codeB2ToB64(b, l): + """ + Returns conversion (encode) of l Base2 sextets from front of b to Base64 chars. + One char for each of l sextets from front (left) of b. + This is useful for encoding as code characters, sextets from the front of + a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 + and l=4. Both require 3 bytes in b. Trailing pad bits are removed so + returned sextets as characters are right aligned . + """ + if hasattr(b, 'encode'): + b = b.encode("utf-8") # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') # convert only first n bytes to int + # check if prepad bits are zero + tbs = 2 * (l % 4) # trailing bit size in bits + i >>= tbs # right shift out trailing bits to make right aligned + return (intToB64(i, l)) # return as B64 + + +def nabSextets(b, l): + """ + Return first l sextets from front (left) of b as bytes (byte string). + Length of bytes returned is minimum sufficient to hold all l sextets. + Last byte returned is right bit padded with zeros + b is bytes or str + """ + if hasattr(b, 'encode'): + b = b.encode() # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') + p = 2 * (l % 4) + i >>= p # strip of last bits + i <<= p # pad with empty bits + return (i.to_bytes(n, 'big')) + + +def keyToKey64u(key): + """ + Returns 64u + Convert and return bytes key to unicode base64 url-file safe version + """ + return base64.urlsafe_b64encode(key).decode("utf-8") + + +def key64uToKey(key64u): + """ + Returns bytes + Convert and return unicode base64 url-file safe key64u to bytes key + """ + return base64.urlsafe_b64decode(key64u.encode("utf-8")) + + +def verifyEd25519(sig, msg, vk): + """ + Returns True if signature sig of message msg is verified with + verification key vk Otherwise False + All of sig, msg, vk are bytes + """ + try: + result = pysodium.crypto_sign_verify_detached(sig, msg, vk) + except Exception as ex: + return False + return (True if result else False) + + +def verify64uEd25519(signature, message, verkey): + """ + Returns True if signature is valid for message with respect to verification + key verkey + + signature and verkey are encoded as unicode base64 url-file strings + and message is unicode string as would be the case for a json object + + """ + sig = key64uToKey(signature) + vk = key64uToKey(verkey) + msg = message.encode("utf-8") + return (verifyEd25519(sig, msg, vk)) + + diff --git a/src/keri/kering.py b/src/keri/kering.py index f56bc4023..2177b1be8 100644 --- a/src/keri/kering.py +++ b/src/keri/kering.py @@ -4,62 +4,164 @@ """ import sys import re -from collections import namedtuple +from collections import namedtuple, deque +from dataclasses import dataclass, astuple +from .help.helping import sceil +from .help.helping import intToB64, intToB64b, b64ToInt + + +MaxON = int("f"*32, 16) # 256 ** 16 - 1 maximum ordinal number, sequence or first seen etc -FALSEY = (False, 0, None, "?0", "no", "false", "False", "off") -TRUTHY = (True, 1, "?1", "yes" "true", "True", 'on') # Serialization Kinds -Serialage = namedtuple("Serialage", 'json mgpk cbor') -Serials = Serialage(json='JSON', mgpk='MGPK', cbor='CBOR') +Kindage = namedtuple("Kindage", 'json mgpk cbor cesr') +Kinds = Kindage(json='JSON', mgpk='MGPK', cbor='CBOR', cesr='CESR') # Protocol Types -Protocolage = namedtuple("Protocolage", "keri crel acdc") -Protos = Protocolage(keri="KERI", crel="CREL", acdc="ACDC", ) +Protocolage = namedtuple("Protocolage", "keri acdc") +Protocols = Protocolage(keri="KERI", acdc="ACDC") Versionage = namedtuple("Versionage", "major minor") Version = Versionage(major=1, minor=0) # KERI Protocol Version Vrsn_1_0 = Versionage(major=1, minor=0) # KERI Protocol Version Specific -Vrsn_1_1 = Versionage(major=1, minor=1) # KERI Protocol Version Specific +Vrsn_2_0 = Versionage(major=2, minor=0) # KERI Protocol Version Specific + -VERRAWSIZE = 6 # hex characters in raw serialization size in version string # "{:0{}x}".format(300, 6) # make num char in hex a variable # '00012c' VERFMT = "{}{:x}{:x}{}{:0{}x}_" # version format string -VERFULLSIZE = 17 # number of characters in full version string +VERRAWSIZE = 6 # hex characters in raw serialization size in version string + +#VEREX0 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' +#Rever = re.compile(VEREX0) # compile is faster + +# version string in JSON, CBOR, or MGPK field map serialization version 1 +VER1FULLSPAN = 17 # number of characters in full version string +VER1TERM = b'_' +VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' + +# version string in JSON, CBOR, or MGPK field map serialization version 2 +VER2FULLSPAN = 16 # number of characters in full version string +VER2TERM = b'.' +VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4})\.' + +VEREX = VEREX2 + b'|' + VEREX1 + +# max number of characters in full version string +MAXVERFULLSPAN = max(VER2FULLSPAN, VER1FULLSPAN) -VEREX = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' Rever = re.compile(VEREX) # compile is faster +MAXVSOFFSET = 12 +SMELLSIZE = MAXVSOFFSET + MAXVERFULLSPAN # min buffer size to inhale + + + +""" +Smellage (results of smelling a version string such as in a Serder) + proto (str): protocol type value of Protocols examples 'KERI', 'ACDC' + vrsn (Versionage): protocol version namedtuple (major, minor) of ints + kind (str): serialization value of Serials examples 'JSON', 'CBOR', 'MGPK' + size (int): int size of raw serialization or + gvrsn (None | Versionage): optional default is None + For CESR native genus version namedtuple (major, minor) of ints + +""" +Smellage = namedtuple("Smellage", "proto vrsn kind size gvrsn", defaults=(None, )) + +def rematch(match): + """ + Returns: + smellage (Smellage): named tuple extracted from version string regex match + (protocol, version, kind, size) + + Parameters: + match (re.Match): instance of Match class + + Notes: + regular expressions work with memoryview objects not just bytes or + bytearrays + """ + full = match.group() # full matched version string + if len(full) == VER2FULLSPAN and full[-1] == ord(VER2TERM): + proto, major, minor, kind, size = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + proto = proto.decode("utf-8") + if proto not in Protocols: + raise ProtocolError(f"Invalid protocol={proto}.") + vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) + if vrsn.major < 2: # version2 vs but major < 2 + raise VersionError(f"Incompatible {vrsn=} with version string.") + + kind = kind.decode("utf-8") + if kind not in Kinds: + raise KindError(f"Invalid serialization kind = {kind}.") + size = b64ToInt(size) + + elif len(full) == VER1FULLSPAN and full[-1] == ord(VER1TERM): + proto, major, minor, kind, size = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + proto = proto.decode("utf-8") + if proto not in Protocols: + raise ProtocolError(f"Invalid protocol={proto}.") + vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) + if vrsn.major > 1: # version1 vs but major > 1 + raise VersionError(f"Incompatible {vrsn=} with version string.") + + kind = kind.decode("utf-8") + if kind not in Kinds: + raise KindError(f"Invalid serialization kind = {kind}.") + size = int(size, 16) + + else: + raise VersionError(f"Bad rematch.") + + return Smellage(proto=proto, vrsn=vrsn, kind=kind, size=size) + -def versify(proto=Protos.keri, version=Version, kind=Serials.json, size=0): +def versify(protocol=Protocols.keri, version=Version, kind=Kinds.json, size=0): """ - Returns version string + Returns: + vs (str): version string + + Parameters: + protocol (str): protocol one of Protocols + version (Versionage): namedtuple (major, minor) of ints + kind (str): one of Serials + size (int): length of serialized map that embeds version string field. """ - if proto not in Protos: - raise ValueError("Invalid message identifier = {}".format(proto)) - #version = version if version else Version - if kind not in Serials: - raise ValueError("Invalid serialization kind = {}".format(kind)) + if protocol not in Protocols: + raise ProtocolError("Invalid message identifier = {}".format(protocol)) + if kind not in Kinds: + raise KindError("Invalid serialization kind = {}".format(kind)) - return VERFMT.format(proto, version[0], version[1], kind, size, VERRAWSIZE) + if version.major < 2: # version1 version string + return VERFMT.format(protocol, version.major, version.minor, kind, size, VERRAWSIZE) + else: # version 2+ version string + return (f"{protocol}{intToB64(version.major)}" + f"{intToB64(version.minor, l=2)}{kind}{intToB64(size, l=4)}.") -def deversify(vs, version=None): +def deversify(vs): """ Returns: tuple(proto, kind, version, size) Where: - proto (str): value is protocol type identifier one of Protos (Protocolage) + proto (str): value is protocol type identifier one of Protocols (Protocolage) acdc='ACDC', keri='KERI' + + vrsn (tuple): version tuple of type Versionage kind (str): value is serialization kind, one of Serials json='JSON', mgpk='MGPK', cbor='CBOR' - vrsn (tuple): version tuple of type Versionage size (int): raw size in bytes Parameters: - vs (str): version string to extract from - version (Versionage | None): supported version. None means do not check - for supported version. + vs (str | bytes): version string to extract from Uses regex match to extract: protocol type @@ -67,28 +169,132 @@ def deversify(vs, version=None): serialization kind serialization size """ - match = Rever.match(vs.encode("utf-8")) # match takes bytes - if match: - proto, major, minor, kind, size = match.group("proto", - "major", - "minor", - "kind", - "size") - proto = proto.decode("utf-8") - vrsn = Versionage(major=int(major, 16), minor=int(minor, 16)) - kind = kind.decode("utf-8") + if hasattr(vs, "encode"): # match takes bytes + vs = vs.encode("utf-8") - if proto not in Protos: - raise ValueError("Invalid message identifier = {}".format(proto)) - if version is not None and vrsn != version: - raise ValueError(f"Expected version = {version}, got " - f"{vrsn.major}.{vrsn.minor}.") - if kind not in Serials: - raise ValueError("Invalid serialization kind = {}".format(kind)) - size = int(size, 16) - return proto, vrsn, kind, size + match = Rever.match(vs) + if not match: + raise VersionError(f"Invalid version string = '{vs}'.") + + return rematch(match) + + +def smell(raw): + """Extract and return instance of Smellage from version string inside + raw serialization. + + Returns: + smellage (Smellage): named Tuple of (protocol, version, kind, size) + + Parameters: + raw (bytearray) of serialized incoming message stream. Assumes start + of stream is JSON, CBOR, or MGPK field map with first field + is labeled 'v' and value is version string. + + + """ + if len(raw) < SMELLSIZE: + raise ShortageError(f"Need more raw bytes to smell full version string.") + + match = Rever.search(raw) # Rever regex takes bytes/bytearray not str + if not match or match.start() > MAXVSOFFSET: + raise VersionError(f"Invalid version string from smelled raw = " + f"{raw[: SMELLSIZE]}.") + + return rematch(match) + + +@dataclass(frozen=True) +class ColdCodex: + """ + ColdCodex is codex of cold stream start tritets of first byte + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + + First three bits: + 0o0 = 000 annotated B64 (exhaustive) + 0o1 = 001 cntcode B64 + 0o2 = 010 opcode B64 + 0o3 = 011 json + 0o4 = 100 mgpk1 + 0o5 = 101 cbor + 0o6 = 110 mgpk2 + 007 = 111 cntcode or opcode B2 + + status is one of ('evt', 'txt', 'bny' ) + 'evt' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) + 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) + 'bny' if tritet in (ColdDex.CtOpB2,) + 'ann' if trited in (ColdDex.AnB64) + + otherwise raise ColdStartError + + x = bytearray([0x2d, 0x5f]) + x == bytearray(b'-_') + x[0] >> 5 == 0o1 + True + """ + AnB64: int = 0o0 # Annotated CESR + CtB64: int = 0o1 # CountCode Base64 + OpB64: int = 0o2 # OpCode Base64 + JSON: int = 0o3 # JSON Map Event Start + MGPK1: int = 0o4 # MGPK Fixed Map Event Start + CBOR: int = 0o5 # CBOR Map Event Start + MGPK2: int = 0o6 # MGPK Big 16 or 32 Map Event Start + CtOpB2: int = 0o7 # CountCode or OpCode Base2 + + def __iter__(self): + return iter(astuple(self)) + + +ColdDex = ColdCodex() # Make instance + +Coldage = namedtuple("Coldage", 'msg txt bny') # stream cold start status +Colds = Coldage(msg='msg', txt='txt', bny='bny') # add 'ant' for annotated + + +def sniff(ims): + """ + Returns status string of cold start of stream ims bytearray by looking + at first triplet of first byte to determin if message or counter code + and if counter code whether Base64 or Base2 representation + + First three bits: + 0o0 = 000 annotated cesr + 0o1 = 001 cntcode B64 + 0o2 = 010 opcode B64 + 0o3 = 011 json + 0o4 = 100 mgpk + 0o5 = 101 cbor + 0o6 = 110 mgpk + 007 = 111 cntcode B2 or opcode B2 + + counter B64 in (0o1, 0o2) return 'txt' + counter B2 in (0o7) return 'bny' + event in (0o3, 0o4, 0o5, 0o6) return 'evt' + unexpected in (0o0) raise ColdStartError + Colds = Coldage(msg='msg', txt='txt', bny='bny') + + 'msg' if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2) + 'txt' if tritet in (ColdDex.CtB64, ColdDex.OpB64) + 'bny' if tritet in (ColdDex.CtOpB2,) + 'ano' if tritet in (ColdDex.Anno,) + """ + if not ims: + raise ShortageError("Need more bytes.") + + tritet = ims[0] >> 5 + if tritet in (ColdDex.JSON, ColdDex.MGPK1, ColdDex.CBOR, ColdDex.MGPK2): + return Colds.msg + if tritet in (ColdDex.CtB64, ColdDex.OpB64): + return Colds.txt + if tritet in (ColdDex.CtOpB2,): + return Colds.bny + #if tritet in (ColdDex.AnB64, ): + + + raise ColdStartError("Unexpected tritet={} at stream start.".format(tritet)) - raise ValueError("Invalid version string = {}".format(vs)) """ ilk is short for packet or message type for a given protocol @@ -111,14 +317,19 @@ def deversify(vs, version=None): brv = backed vc revoke, registry-backed transaction event log credential revocation """ -# KERI protocol packet (message) types -Ilkage = namedtuple("Ilkage", ('icp rot ixn dip drt rct qry rpy exn ' - 'pro bar vcp vrt iss rev bis brv ')) +# KERI/ACDC protocol packet (message) types +Ilkage = namedtuple("Ilkage", ('icp rot ixn dip drt rct qry rpy xip exn ' + 'pro bar vcp vrt iss rev bis brv rip upd ' + 'acd ace sch att agg edg rul ')) Ilks = Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', rct='rct', - qry='qry', rpy='rpy', exn='exn', pro='pro', bar='bar', - vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv') + qry='qry', rpy='rpy', xip='xip', exn='exn', pro='pro', bar='bar', + vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv', + rip='rip', upd='upd', acd='acd', ace='ace', + sch='sch', att='att', agg='agg', edg='edg', rul='rul') + +# Ilks needs to be versioned for Protocol versions or else use Serder.Fields # note ksn is not actual standalone message but is embedded in exn msg when sent # over the wire. But keep ilk for legacy reasons. @@ -136,34 +347,30 @@ def deversify(vs, version=None): watcher='watcher', judge='judge', juror='juror', peer='peer', mailbox="mailbox", agent="agent") -ICP_LABELS = ["v", "t", "d", "i", "s", "kt", "k", "nt", "n", - "bt", "b", "c", "a"] -DIP_LABELS = ["v", "d", "i", "s", "t", "kt", "k", "nt", "n", - "bt", "b", "c", "a", "di"] -ROT_LABELS = ["v", "d", "i", "s", "t", "p", "kt", "k", "nt", "n", - "bt", "br", "ba", "a"] -DRT_LABELS = ["v", "d", "i", "s", "t", "p", "kt", "k", "nt", "n", - "bt", "br", "ba", "a"] -IXN_LABELS = ["v", "d", "i", "s", "t", "p", "a"] - -#KSN_LABELS = ["v", "d", "i", "s", "p", "d", "f", "dt", "et", "kt", "k", "nt", "n", - #"bt", "b", "c", "ee", "di"] - -RPY_LABELS = ["v", "d", "t", "d", "dt", "r", "a"] +@dataclass(frozen=True) +class TraitCodex: + """ + TraitCodex is codex of inception configuration trait code strings + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. -VCP_LABELS = ["v", "d", "i", "s", "t", "bt", "b", "c"] -VRT_LABELS = ["v", "d", "i", "s", "t", "p", "bt", "b", "ba", "br"] + """ + EstOnly: str = 'EO' # Only allow establishment events. Inception only. + DoNotDelegate: str = 'DND' # Dot not allow delegated identifiers. Inception only. + RegistrarBackers: str = 'RB' # Registrar backer provided in Registrar seal in this event + NoBackers: str = 'NB' # Do not allow any (registrar backers). + # Inception and Rotation in v2. This should be NRB in next version. + NoRegistrarBackers: str = 'NRB' # Do not allow any registrar backers. Inception and Rotation. + DelegateIsDelegator: str = 'DID' # Treat delegate AIDs same as their delegator. Inception only -ISS_LABELS = ["v", "i", "s", "t", "ri", "dt"] -BIS_LABELS = ["v", "i", "s", "t", "ra", "dt"] + def __iter__(self): + return iter(astuple(self)) -REV_LABELS = ["v", "i", "s", "t", "p", "dt"] -BRV_LABELS = ["v", "i", "s", "t", "ra", "p", "dt"] -TSN_LABELS = ["v", "i", "s", "d", "ii", "a", "et", "bt", "b", "c", "br", "ba"] -CRED_TSN_LABELS = ["v", "i", "s", "d", "ri", "a", "ra"] +TraitDex = TraitCodex() # Make instance +# Exception Subclasses class KeriError(Exception): """ Base Class for keri exceptions @@ -254,9 +461,17 @@ class MaterialError(KeriError): class RawMaterialError(MaterialError): """ - Not Enough bytes in buffer bytearray for raw material + Invalid raw material Usage: - raise ShortageError("error message") + raise RawMaterialError("error message") + """ + + +class SoftMaterialError(MaterialError): + """ + Invalid soft material + Usage: + raise SoftMaterialError("error message") """ @@ -268,6 +483,15 @@ class EmptyMaterialError(MaterialError): """ +class InvalidVersionError(MaterialError): + """ + Invalid, Unknown, or unrecognized CESR code table version encountered during + crypto material init + Usage: + raise InvalidVersionError("error message") + """ + + class InvalidCodeError(MaterialError): """ Invalid, Unknown, or unrecognized code encountered during crypto material init @@ -275,6 +499,15 @@ class InvalidCodeError(MaterialError): raise InvalidCodeError("error message") """ + +class InvalidSoftError(MaterialError): + """ + Invalid, Unknown, or unrecognized soft part encountered during crypto material init + Usage: + raise InvalidSoftError("error message") + """ + + class InvalidTypeError(MaterialError): """ Invalid material value type encountered during crypto material init @@ -282,6 +515,7 @@ class InvalidTypeError(MaterialError): raise InvalidTypeError("error message") """ + class InvalidValueError(MaterialError): """ Invalid material value encountered during crypto material init @@ -289,6 +523,7 @@ class InvalidValueError(MaterialError): raise InvalidValueError("error message") """ + class InvalidSizeError(MaterialError): """ Invalid size encountered during crypto material init @@ -348,6 +583,7 @@ class ValidationError(KeriError): raise ValidationError("error message") """ + class MissingFieldError(ValidationError): """ Missing a required element or field of message @@ -356,6 +592,23 @@ class MissingFieldError(ValidationError): """ +class ExtraFieldError(ValidationError): + """ + Extra unallowed field in message + Usage: + raise ExtraFieldError("error message") + """ + + +class AlternateFieldError(ValidationError): + """ + Unallowed alternate field in message + + Usage: + raise AlternateFieldError("error message") + """ + + class MissingSignatureError(ValidationError): """ Error At least One but Missing Enough Signatures for Threshold @@ -504,6 +757,21 @@ class OutOfOrderTxnStateError(ValidationError): raise OutOfOrderTxnStateError("error message") """ +class MisfitEventSourceError(ValidationError): + """ + Error referenced event missing from log so can't verify this txn state event + Usage: + raise MisfitEventSourceError("error message") + """ + +class MissingDelegableApprovalError(ValidationError): + """ + Error referenced event missing from log so can't verify this txn state event + Usage: + raise MissingDelegableApprovalError("error message") + """ + + # Stream Parsing and Extraction Errors class ExtractionError(KeriError): @@ -565,6 +833,13 @@ class KindError(ExtractionError): raise KindError("error message") """ +class IlkError(ExtractionError): + """ + Bad or Unsupported Message Type (Ilk) + + Usage: + raise IlkError("error message") + """ class ConversionError(ExtractionError): """ diff --git a/src/keri/metric/__init__.py b/src/keri/metric/__init__.py new file mode 100644 index 000000000..bf9cd378c --- /dev/null +++ b/src/keri/metric/__init__.py @@ -0,0 +1,7 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.metric Package +""" + +from .metricing import EscrowEnd diff --git a/src/keri/metric/metricing.py b/src/keri/metric/metricing.py new file mode 100644 index 000000000..e4f76bd7c --- /dev/null +++ b/src/keri/metric/metricing.py @@ -0,0 +1,95 @@ +# -*- encoding: utf-8 -*- +""" +KERI +keri.metric.metricing module + +Prometheus metrics endpoints for KERI escrow monitoring +""" +import falcon + + +class EscrowEnd: + """Prometheus metrics endpoint for escrow counts. + + Exposes escrow counts in Prometheus text format for monitoring/alerting. + """ + + def __init__(self, hby, reger): + """Initialize EscrowEnd. + + Parameters: + hby (Habery): Habery instance for accessing KEL escrows + reger (Reger): Registry for accessing TEL escrows + """ + self.hby = hby + self.reger = reger + + def on_get(self, _, rep): + """GET /metrics - Returns Prometheus format metrics. + + Parameters: + _ (Request): Falcon HTTP request object (unused) + rep (Response): Falcon HTTP response object + """ + lines = [] + + # Header + lines.append("# HELP keri_escrow_count Number of items in each escrow type") + lines.append("# TYPE keri_escrow_count gauge") + + # KEL / Baser escrows + escrow_counts = [ + ("unverified_receipts", sum(1 for _ in self.hby.db.getUreItemIter())), + ("verified_receipts", sum(1 for _ in self.hby.db.getVreItemIter())), + ("out_of_order_events", sum(1 for _ in self.hby.db.getOoeItemIter())), + ("partially_witnessed_events", sum(1 for _ in self.hby.db.getPweItemIter())), + ("partially_signed_events", sum(1 for _ in self.hby.db.getPseItemIter())), + ("likely_duplicitous_events", sum(1 for _ in self.hby.db.getLdeItemIter())), + ("unverified_event_indexed_couples", sum(1 for _ in self.hby.db.getUweItemIter())), + ("query_not_found", sum(1 for _ in self.hby.db.qnfs.getItemIter())), + ("partially_delegated_events", sum(1 for _ in self.hby.db.pdes.getItemIter())), + ("reply", sum(1 for _ in self.hby.db.rpes.getItemIter())), + ("failed_oobi", sum(1 for _ in self.hby.db.eoobi.getItemIter())), + ("group_partial_witness", sum(1 for _ in self.hby.db.gpwe.getItemIter())), + ("group_delegate", sum(1 for _ in self.hby.db.gdee.getItemIter())), + ("delegated_partial_witness", sum(1 for _ in self.hby.db.dpwe.getItemIter())), + ("group_partial_signed", sum(1 for _ in self.hby.db.gpse.getItemIter())), + ("exchange_partial_signed", sum(1 for _ in self.hby.db.epse.getItemIter())), + ("delegated_unanchored", sum(1 for _ in self.hby.db.dune.getItemIter())), + ] + + for name, count in escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="kel"}} {count}') + + # TEL / Reger escrows + tel_escrow_counts = [ + ("out_of_order", sum(1 for _ in self.reger.getOotItemIter())), + ("partially_witnessed", sum(1 for _ in self.reger.getTweItemIter())), + ("anchorless", sum(1 for _ in self.reger.getTaeItemIter())), + ("missing_registry", sum(1 for _ in self.reger.mre.getItemIter())), + ("broken_chain", sum(1 for _ in self.reger.mce.getItemIter())), + ("missing_schema", sum(1 for _ in self.reger.mse.getItemIter())), + ("missing_signature", sum(1 for _ in self.reger.cmse.getItemIter())), + ("partial_witness", sum(1 for _ in self.reger.tpwe.getItemIter())), + ("multisig", sum(1 for _ in self.reger.tmse.getItemIter())), + ("event_dissemination", sum(1 for _ in self.reger.tede.getItemIter())), + ] + + for name, count in tel_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="tel"}} {count}') + + # Registry transaction escrows + registry_escrow_counts = [ + ("registry_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-mae", "")))), + ("registry_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("registry-ooo", "")))), + ("credential_missing_registry", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mre", "")))), + ("credential_missing_anchor", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-mae", "")))), + ("credential_out_of_order", sum(1 for _ in self.reger.txnsb.escrowdb.getItemIter(keys=("credential-ooo", "")))), + ] + + for name, count in registry_escrow_counts: + lines.append(f'keri_escrow_count{{type="{name}",layer="registry"}} {count}') + + rep.content_type = "text/plain; version=0.0.4; charset=utf-8" + rep.status = falcon.HTTP_200 + rep.text = "\n".join(lines) + "\n" diff --git a/src/keri/peer/exchanging.py b/src/keri/peer/exchanging.py index 662cfef54..e9222afc9 100644 --- a/src/keri/peer/exchanging.py +++ b/src/keri/peer/exchanging.py @@ -3,12 +3,13 @@ keri.peer.exchanging module """ +import datetime import logging from datetime import timedelta from hio.help import decking -from .. import help, kering +from .. import help, kering, core from ..app import habbing from ..core import eventing, coring, serdering from ..help import helping @@ -24,6 +25,8 @@ class Exchanger: Peer to Peer KERI message Exchanger. """ + TimeoutPSE = 20 # seconds to timeout partially signed or delegated escrows + def __init__(self, hby, handlers, cues=None, delta=ExchangeMessageTimeWindow): """ Initialize instance @@ -71,19 +74,25 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): route = serder.ked["r"] sender = serder.ked["i"] pathed = kwargs["pathed"] if "pathed" in kwargs else [] + essrs = kwargs["essrs"] if "essrs" in kwargs else [] behavior = self.routes[route] if route in self.routes else None - if tsgs is not None: + if tsgs: for prefixer, seqner, ssaider, sigers in tsgs: # iterate over each tsg if sender != prefixer.qb64: # sig not by aid - raise MissingSignatureError(f"Exchange process: skipped signature not from aid=" - f"{sender}, from {prefixer.qb64} on exn msg=\n{serder.pretty()}\n") + msg = (f"Skipped signature not from aid = " + f"{sender}, from {prefixer.qb64} on exn msg = {serder.said}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) if prefixer.qb64 not in self.kevers or self.kevers[prefixer.qb64].sn < seqner.sn: if self.escrowPSEvent(serder=serder, tsgs=tsgs, pathed=pathed): self.cues.append(dict(kin="query", q=dict(r="logs", pre=prefixer.qb64, sn=seqner.snh))) - raise MissingSignatureError(f"Unable to find sender {prefixer.qb64} in kevers" - f" for evt = {serder.ked}.") + msg = f"Unable to find sender {prefixer.qb64} in kevers for evt = {serder.said}" + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) # Verify the signatures are valid and that the signature threshold as of the signing event is met tholder, verfers = self.hby.db.resolveVerifiers(pre=prefixer.qb64, sn=seqner.sn, dig=ssaider.qb64) @@ -92,26 +101,39 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): if not tholder.satisfy(indices): # We still don't have all the sigers, need to escrow if self.escrowPSEvent(serder=serder, tsgs=tsgs, pathed=pathed): self.cues.append(dict(kin="query", q=dict(r="logs", pre=prefixer.qb64, sn=seqner.snh))) - msg = f"Not enough signatures in {indices} for evt = {serder.said}" + msg = (f"Not enough signatures in idx={indices} route={route} " + f"for evt = {serder.said} recipient={serder.ked.get('rp', '')}") logger.info(msg) - logger.debug(f"Event body=\n%20\n", serder.pretty()) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) raise MissingSignatureError(msg) - elif cigars is not None: + elif cigars: for cigar in cigars: if sender != cigar.verfer.qb64: # cig not by aid - raise MissingSignatureError(" process: skipped cig not from aid=" - "%s on exn msg=\n%s\n", sender, serder.pretty()) + msg = (f"Skipped cig not from aid={sender} route={route} " + f"for exn evt = {serder.said} recipient={serder.ked.get('rp', '')}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) if not cigar.verfer.verify(cigar.raw, serder.raw): # cig not verify - raise MissingSignatureError("Failure satisfying exn on cigs for {}" - " for evt = {}.".format(cigar, - serder.ked)) + msg = (f"Failure satisfying exn on cigs for {cigar} route={route} " + f"for evt = {serder.said} recipient={serder.ked.get('rp', '')}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) else: - raise MissingSignatureError("Failure satisfying exn, no cigs or sigs" - " for evt = {}.".format(serder.ked)) + self.escrowPSEvent(serder=serder, tsgs=[], pathed=pathed) + msg = ( + f"Failure satisfying exn, no cigs or sigs for evt = {serder.said} " + f"on route {route} recipient = {serder.ked.get('rp', '')}") + logger.info(msg) + logger.debug("Exchange message body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) e = coring.Pather(path=["e"]) + + kwargs = dict() attachments = [] for p in pathed: pattach = bytearray(p) @@ -120,27 +142,41 @@ def processEvent(self, serder, tsgs=None, cigars=None, **kwargs): np = pather.strip(e) attachments.append((np, pattach)) + kwargs["attachments"] = attachments + if essrs: + kwargs["essr"] = b''.join([texter.raw for texter in essrs]) + + if isinstance(serder.seals, str): + if 'essr' not in kwargs: + raise ValidationError("at least one essr attachment is required") + + essr = kwargs['essr'] + dig = serder.seals + diger = coring.Diger(qb64=dig) + if not diger.verify(ser=essr): + raise ValidationError(f"essr diger={diger.qb64} is invalid against content") + # Perform behavior specific verification, think IPEX chaining requirements try: - if not behavior.verify(serder=serder, attachments=attachments): - logger.info(f"exn event for route {route} failed behavior verification. exn={serder.said}") - logger.debug(f"exn body=\n{serder.ked}\n") + if not behavior.verify(serder=serder, **kwargs): + logger.error("exn event for route %s failed behavior verification. said=%s", route, serder.said) + logger.debug(f"Event=\n%s\n", serder.pretty()) return + except AttributeError: - logger.debug(f"Behavior for {route} missing or does not have verify for exn={serder.said}") - logger.debug(f"exn Body=\n{serder.ked}\n") + logger.debug("Behavior for %s missing or does not have verify for said %s", route, serder.said) + logger.debug("Exn Event Body=\n%s\n", serder.pretty()) - # Always persis events - self.logEvent(serder, pathed, tsgs, cigars) + # Always persist events + self.logEvent(serder, pathed, tsgs, cigars, essrs) self.cues.append(dict(kin="saved", said=serder.said)) # Execute any behavior specific handling, not sure if this should be different than verify try: - behavior.handle(serder=serder, attachments=attachments) + behavior.handle(serder=serder, **kwargs) except AttributeError: - logger.debug(f"Behavior for {route} missing or does not have handle for exn={serder.said}") - logger.debug( - f"exn body=\n{serder.ked}\n") + logger.debug("Behavior for %s missing or does not have handle for SAID=%s", route, serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) def processEscrow(self): """ Process all escrows for `exn` messages @@ -163,57 +199,80 @@ def escrowPSEvent(self, serder, tsgs, pathed): for siger in sigers: self.hby.db.esigs.add(keys=quadkeys, val=siger) + self.hby.db.epsd.put(keys=(dig,), val=coring.Dater()) self.hby.db.epath.pin(keys=(dig,), vals=[bytes(p) for p in pathed]) return self.hby.db.epse.put(keys=(dig,), val=serder) def processEscrowPartialSigned(self): """ Process escrow of partially signed messages """ for (dig,), serder in self.hby.db.epse.getItemIter(): - tsgs = [] - klases = (coring.Prefixer, coring.Seqner, coring.Saider) - args = ("qb64", "snh", "qb64") - sigers = [] - old = None # empty keys - for keys, siger in self.hby.db.esigs.getItemIter(keys=(dig, "")): - quad = keys[1:] - if quad != old: # new tsg - if sigers: # append tsg made for old and sigers - prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) - - tsgs.append((prefixer, seqner, saider, sigers)) - sigers = [] - old = quad - sigers.append(siger) - if sigers and old: - prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) - tsgs.append((prefixer, seqner, saider, sigers)) + try: + tsgs = [] + klases = (coring.Prefixer, coring.Seqner, coring.Saider) + args = ("qb64", "snh", "qb64") + sigers = [] - pathed = [bytearray(p.encode("utf-8")) for p in self.hby.db.epath.get(keys=(dig,))] + dtnow = helping.nowUTC() + dater = self.hby.db.epsd.get(keys=(dig,)) + if dater is None: + raise ValidationError("Missing exn escrowed event datetime " + f"at dig = {dig}.") + + dte = dater.datetime + if (dtnow - dte) > datetime.timedelta(seconds=self.TimeoutPSE): + # escrow stale so raise ValidationError which unescrows below + raise ValidationError("Stale exn event escrow " + f"at dig = {dig}.") + + old = None # empty keys + for keys, siger in self.hby.db.esigs.getItemIter(keys=(dig, "")): + quad = keys[1:] + if quad != old: # new tsg + if sigers: # append tsg made for old and sigers + prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) + + tsgs.append((prefixer, seqner, saider, sigers)) + sigers = [] + old = quad + sigers.append(siger) + if sigers and old: + prefixer, seqner, saider = helping.klasify(sers=old, klases=klases, args=args) + tsgs.append((prefixer, seqner, saider, sigers)) + + pathed = [bytearray(p.encode("utf-8")) for p in self.hby.db.epath.get(keys=(dig,))] + essrs = [texter for texter in self.hby.db.essrs.get(keys=(dig,))] + + kwargs = dict() + if essrs: + kwargs["essrs"] = essrs + self.processEvent(serder=serder, tsgs=tsgs, pathed=pathed, **kwargs) - try: - self.processEvent(serder=serder, tsgs=tsgs, pathed=pathed) except MissingSignatureError as ex: - logger.trace("Exchange partially signed unescrow failed: %s\n", ex.args[0]) if logger.isEnabledFor(logging.TRACE): - logger.debug("Event body=\n%s\n", serder.pretty()) + logger.trace("Exchange partially signed unescrow failed: %s\n", ex.args[0]) + logger.debug(f"Event body=\n%s\n", serder.pretty()) except Exception as ex: self.hby.db.epse.rem(dig) + self.hby.db.epsd.rem(dig) self.hby.db.esigs.rem(dig) - logger.info("Exchange partially signed unescrowed: %s\n", ex.args[0]) if logger.isEnabledFor(logging.DEBUG): - logger.debug("Event body=\n%s\n", serder.pretty()) + logger.exception("Exchanger: partially signed unescrowed: %s", ex.args[0]) + else: + logger.error("Exchanger: partially signed unescrowed: %s", ex.args[0]) else: self.hby.db.epse.rem(dig) self.hby.db.esigs.rem(dig) - logger.info("Exchanger unescrow succeeded in valid exchange: serder = %s", serder.said) - logger.debug("Serder Body=\n%s\n", serder.pretty()) + logger.info("Exchanger: unescrow succeeded in valid exchange: " + "creder=%s", serder.said) + logger.debug("Event=\n%s\n", serder.pretty()) - def logEvent(self, serder, pathed=None, tsgs=None, cigars=None): + def logEvent(self, serder, pathed=None, tsgs=None, cigars=None, essrs=None): dig = serder.said pdig = serder.ked['p'] pathed = pathed or [] tsgs = tsgs or [] cigars = cigars or [] + essrs = essrs or [] for prefixer, seqner, ssaider, sigers in tsgs: # iterate over each tsg quadkeys = (serder.said, prefixer.qb64, f"{seqner.sn:032x}", ssaider.qb64) @@ -224,19 +283,12 @@ def logEvent(self, serder, pathed=None, tsgs=None, cigars=None): saider = coring.Saider(qb64=serder.said) self.hby.db.epath.pin(keys=(dig,), vals=[bytes(p) for p in pathed]) + for texter in essrs: + self.hby.db.essrs.add(keys=(dig,), val=texter) if pdig: self.hby.db.erpy.pin(keys=(pdig,), val=saider) self.hby.db.exns.put(keys=(dig,), val=serder) - # special handling for 1.1.18 -> 1.1.32 - recipient = None - if 'rp' in serder.ked: - recipient = serder.ked['rp'] - sender = serder.ked['i'] - route = serder.ked['r'] - logger.info("Saved exn event route = %s SAID = %s sender %s -> recipient %s", - route, dig, sender, recipient) - logger.debug("EXN Event Body=\n%s\n", serder.pretty()) def lead(self, hab, said): """ Determines is current member represented by hab is the lead of an exn message @@ -285,20 +337,22 @@ def complete(self, said): def exchange(route, - payload, sender, + payload=None, + diger=None, recipient=None, date=None, dig=None, modifiers=None, embeds=None, version=coring.Version, - kind=coring.Serials.json): + kind=coring.Kinds.json): """ Create an `exn` message with the specified route and payload Parameters: route (str): to destination route of the message payload (list | dict): body of message to deliver to route + diger (Diger): qb64 digest of payload sender (str): qb64 AID of sender of the exn recipient (str) optional qb64 AID recipient of exn date (str): Iso8601 formatted date string to use for this request @@ -314,6 +368,7 @@ def exchange(route, ilk = eventing.Ilks.exn dt = date if date is not None else helping.nowIso8601() p = dig if dig is not None else "" + rp = recipient if recipient is not None else "" embeds = embeds if embeds is not None else {} e = dict() @@ -329,27 +384,40 @@ def exchange(route, pather = coring.Pather(path=["e", label]) pathed.extend(pather.qb64b) pathed.extend(atc) - end.extend(coring.Counter(code=coring.CtrDex.PathedMaterialQuadlets, - count=(len(pathed) // 4)).qb64b) + if len(pathed) // 4 < 4096: + end.extend(core.Counter(core.Codens.PathedMaterialGroup, + count=(len(pathed) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) + else: + end.extend(core.Counter(core.Codens.BigPathedMaterialGroup, + count=(len(pathed) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) end.extend(pathed) if e: e["d"] = "" _, e = coring.Saider.saidify(sad=e, label=coring.Saids.d) - attrs = dict( - ) + modifiers = modifiers if modifiers is not None else {} + + if diger is None: + attrs = dict() + + if recipient is not None: + attrs['i'] = recipient - if recipient is not None: - attrs['i'] = recipient + attrs |= payload - attrs |= payload + else: + attrs = diger.qb64 + # Attr field 'a' can be either a said or a nested block and the fields + # of the nested block can be saids of further nested block or nested blocks ked = dict(v=vs, t=ilk, d="", i=sender, - rp="", + rp=rp, p=p, dt=dt, r=route, @@ -403,17 +471,20 @@ def serializeMessage(hby, said, pipelined=False): if len(tsgs) > 0: for (prefixer, seqner, saider, sigers) in tsgs: - atc.extend(coring.Counter(coring.CtrDex.TransIdxSigGroups, count=1).qb64b) + atc.extend(core.Counter(core.Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) - atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + atc.extend(core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: atc.extend(siger.qb64b) if len(cigars) > 0: - atc.extend(coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=len(cigars)).qb64b) + atc.extend(core.Counter(core.Codens.NonTransReceiptCouples, + count=len(cigars), gvrsn=kering.Vrsn_1_0).qb64b) for cigar in cigars: if cigar.verfer.code not in coring.NonTransDex: raise ValueError("Attempt to use tranferable prefix={} for " @@ -423,8 +494,8 @@ def serializeMessage(hby, said, pipelined=False): # Smash the pathed components on the end for p in hby.db.epath.get(keys=(exn.said,)): - atc.extend(coring.Counter(code=coring.CtrDex.PathedMaterialQuadlets, - count=(len(p) // 4)).qb64b) + atc.extend(core.Counter(core.Codens.PathedMaterialGroup, + count=(len(p) // 4), gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(p.encode("utf-8")) msg = bytearray() @@ -433,8 +504,8 @@ def serializeMessage(hby, said, pipelined=False): if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - msg.extend(coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, - count=(len(atc) // 4)).qb64b) + msg.extend(core.Counter(core.Codens.AttachmentGroup, + count=(len(atc) // 4), gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(atc) return msg @@ -483,29 +554,35 @@ def verify(hby, serder): accepted = False for prefixer, seqner, ssaider, sigers in tsgs: if prefixer.qb64 not in hby.kevers or hby.kevers[prefixer.qb64].sn < seqner.sn: - raise MissingSignatureError(f"Unable to find sender {prefixer.qb64} in kevers" - f" for evt = {serder.ked}.") + msg = f"Unable to find sender {prefixer.qb64} in kevers for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) # Verify the signatures are valid and that the signature threshold as of the signing event is met tholder, verfers = hby.db.resolveVerifiers(pre=prefixer.qb64, sn=seqner.sn, dig=ssaider.qb64) _, indices = eventing.verifySigs(serder.raw, sigers, verfers) if not tholder.satisfy(indices): # We still don't have all the sigers, need to escrow - msg = f"Not enough signatures in {indices} for evt = {serder.said}" - logger.info("exchanging.verify: %s", msg) - logger.debug(f"Event body=\n%s\n", serder.pretty()) + msg = f"Not enough signatures in idx={indices} for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) raise MissingSignatureError(msg) accepted = True cigars = hby.db.ecigs.get(keys=(serder.said,)) for cigar in cigars: if not cigar.verfer.verify(cigar.raw, serder.raw): # cig not verify - raise MissingSignatureError("Failure satisfying exn on cigs for {}" - " for evt = {}.".format(cigar, - serder.ked)) + msg = f"Failure satisfying exn on cigs for {cigar} for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) accepted = True if not accepted: - raise MissingSignatureError(f"No valid signatures stored for evt = {serder.ked}") + msg = f"No valid signatures stored for evt = {serder.said}" + logger.info(msg) + logger.debug("Exn Body=\n%s\n", serder.pretty()) + raise MissingSignatureError(msg) return tsgs, cigars diff --git a/src/keri/vc/proving.py b/src/keri/vc/proving.py index d99f85b4c..5f92717fa 100644 --- a/src/keri/vc/proving.py +++ b/src/keri/vc/proving.py @@ -4,55 +4,54 @@ """ -from collections.abc import Iterable -from typing import Union, Optional +from typing import Optional, Union +from .. import core from .. import help from ..core import coring, serdering -from ..core.coring import (Serials, versify) -from ..db import subing -from ..kering import Version +from ..core.coring import (Kinds, versify) from ..help import helping +from ..kering import Version KERI_REGISTRY_TYPE = "KERICredentialRegistry" logger = help.ogler.getLogger() -def credential(schema, - issuer, - data, - recipient=None, - private: bool = False, - private_credential_nonce: Optional[str] = None, - private_subject_nonce: Optional[str] = None, - status=None, - source=None, - rules=None, - version=Version, - kind=Serials.json): +def credential(schema:str, + issuer:str, + data:dict, + recipient:Optional[str]=None, + private:bool=False, + private_credential_nonce:Optional[str]=None, + private_subject_nonce:Optional[str]=None, + status:str=None, + source:Union[dict, list]=None, + rules:Union[dict, list]=None, + version:Version=Version, + kind:Kinds=Kinds.json): """Utility function to create an ACDC. Creates dict SAD for credential from parameters and Saidifyies it before creation. Parameters: - schema (SAID): of schema for this credential + schema (str): SAID of schema for this credential issuer (str): qb64 identifier prefix of the issuer - status (str): qb64 said of the credential registry - recipient (Option[str|None]): qb64 identifier prefix of the recipient data (dict): of the values being assigned to the subject of this credential + recipient (Optional[str]): qb64 identifier prefix of the recipient private (bool): apply nonce used for privacy preserving ACDC private_credential_nonce (Optional[str]): nonce used for privacy vc private_subject_nonce (Optional[str]): nonce used for subject - source (dict | list): of source credentials to which this credential is chained - rules (dict | list): ACDC rules section for credential + status (str): qb64 said of the credential registry + source (Union[dict, list]): of source credentials to which this credential is chained + rules (Union[dict, list]): ACDC rules section for credential version (Version): version instance - kind (Serials): serialization kind + kind (Kinds): serialization kind Returns: - SerderACDC: credential instance + serdering.SerderACDC: credential instance """ - vs = versify(proto=coring.Protos.acdc, version=version, kind=kind, size=0) + vs = versify(protocol=coring.Protocols.acdc, version=version, kind=kind, size=0) vc = dict( v=vs, @@ -64,8 +63,8 @@ def credential(schema, ) if private: - vc["u"] = private_credential_nonce if private_credential_nonce is not None else coring.Salter().qb64 - subject["u"] = private_subject_nonce if private_subject_nonce is not None else coring.Salter().qb64 + vc["u"] = private_credential_nonce if private_credential_nonce is not None else core.Salter().qb64 + subject["u"] = private_subject_nonce if private_subject_nonce is not None else core.Salter().qb64 if recipient is not None: subject['i'] = recipient diff --git a/src/keri/vc/walleting.py b/src/keri/vc/walleting.py index f4ea25b46..0696459e1 100644 --- a/src/keri/vc/walleting.py +++ b/src/keri/vc/walleting.py @@ -72,7 +72,7 @@ def __init__(self, hby, verifier, **kwa): super(WalletDoer, self).__init__(doers=doers, **kwa) - def escrowDo(self, tymth, tock=0.0): + def escrowDo(self, tymth, tock=0.0, **kwa): """ Processes the escrows for group icp, rot and ixn request messages. Parameters: @@ -96,7 +96,7 @@ def escrowDo(self, tymth, tock=0.0): self.verifier.processEscrows() yield self.tock - def verifierDo(self, tymth, tock=0.0): + def verifierDo(self, tymth, tock=0.0, **kwa): """ Process cues from Verifier coroutine Parameters: diff --git a/src/keri/vdr/credentialing.py b/src/keri/vdr/credentialing.py index 3b9632995..212e3ee9e 100644 --- a/src/keri/vdr/credentialing.py +++ b/src/keri/vdr/credentialing.py @@ -11,7 +11,8 @@ from hio.help import decking from keri.vdr import viring -from .. import kering, help +from .. import help +from .. import kering, core from ..app import agenting from ..app.habbing import GroupHab from ..core import parsing, coring, scheming, serdering @@ -64,18 +65,16 @@ def loadRegistries(self): regk = regord.registryKey pre = regord.prefix - # Hot fix, for deleted AIDs - if pre in self.hby.habs: - hab = self.hby.habs[pre] - if hab is None: - raise kering.ConfigurationError(f"Unknown prefix {pre} for creating Registry {name}") + hab = self.hby.habs[pre] + if hab is None: + raise kering.ConfigurationError(f"Unknown prefix {pre} for creating Registry {name}") - reg = Registry(hab=hab, reger=self.reger, tvy=self.tvy, psr=self.psr, - name=name, regk=regk, cues=self.cues) + reg = Registry(hab=hab, reger=self.reger, tvy=self.tvy, psr=self.psr, + name=name, regk=regk, cues=self.cues) - reg.inited = True - self.regs[regk] = reg - self.reger.registries.add(regk) + reg.inited = True + self.regs[regk] = reg + self.reger.registries.add(regk) def makeRegistry(self, name, prefix, **kwa): hab = self.hby.habs[prefix] @@ -236,8 +235,8 @@ def processEvent(self, serder): try: self.tvy.processEvent(serder=serder) except kering.MissingAnchorError: - logger.info("Credential registry missing anchor for inception = {}".format(serder.said)) - logger.debug("Inception body = {}".format(serder.pretty())) + logger.info("Credential registry missing anchor for inception = %s", serder.said) + logger.debug("event=\n%s\n", serder.pretty()) def anchorMsg(self, pre, regd, seqner, saider): """ Create key event with seal to serder anchored as data. @@ -653,7 +652,7 @@ def complete(self, pre, sn=0): said = self.rgy.reger.ctel.get(keys=(pre, seqner.qb64)) return said is not None and self.witPub.sent(said=pre) - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be compeleted. Steps involve: @@ -879,7 +878,7 @@ def processCredentialMissingSigEscrow(self): def complete(self, said): return self.rgy.reger.ccrd.get(keys=(said,)) is not None - def escrowDo(self, tymth, tock=1.0): + def escrowDo(self, tymth, tock=1.0, **kwa): """ Process escrows of group multisig identifiers waiting to be completed. Steps involve: @@ -940,7 +939,8 @@ def sendCredential(hby, hab, reger, postman, creder, recp): postman.send(serder=source, attachment=atc) serder, prefixer, seqner, saider = reger.cloneCred(creder.said) - atc = bytearray(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + atc = bytearray(core.Counter(core.Codens.SealSourceTriples, + count=1, gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) @@ -994,7 +994,7 @@ def sendArtifacts(hby, reger, postman, creder, recp): postman.send(serder=serder, attachment=atc) for msg in reger.clonePreIter(pre=creder.said): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) @@ -1008,16 +1008,16 @@ def sendRegistry(hby, reger, postman, creder, sender, recp): ikever = hby.db.kevers[issr] for msg in hby.db.cloneDelegation(ikever): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) for msg in hby.db.clonePreIter(pre=issr): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) for msg in reger.clonePreIter(pre=regk): - serder = serdering.SerderKERI(raw=msg) # coring.Serder(raw=msg) + serder = serdering.SerderKERI(raw=msg) atc = msg[serder.size:] postman.send(serder=serder, attachment=atc) diff --git a/src/keri/vdr/eventing.py b/src/keri/vdr/eventing.py index 67914b141..3f008bae2 100644 --- a/src/keri/vdr/eventing.py +++ b/src/keri/vdr/eventing.py @@ -17,17 +17,16 @@ from keri import kering from .. import core from .. import help -from ..core import serdering, coring -from ..core.coring import (MtrDex, Serials, versify, Prefixer, - Ilks, Seqner, Verfer) -from ..core.eventing import SealEvent, ample, TraitDex, verifySigs, validateSN +from ..core import serdering, coring, indexing +from ..core.coring import (MtrDex, Kinds, versify, Prefixer, + Ilks, Seqner, Verfer, Number) +from ..core.signing import (Salter,) +from ..core.eventing import SealEvent, ample, TraitDex, verifySigs from ..db import basing, dbing -from ..db.dbing import dgKey, snKey +from ..db.dbing import dgKey, snKey, splitSnKey from ..help import helping from ..kering import (MissingWitnessSignatureError, Version, MissingAnchorError, ValidationError, OutOfOrderError, LikelyDuplicitousError) -from ..kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, - BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) from ..vdr import viring logger = help.ogler.getLogger() @@ -40,7 +39,7 @@ def incept( nonce=None, cnfg=None, version=Version, - kind=Serials.json, + kind=Kinds.json, code=MtrDex.Blake3_256, ): """ Returns serder of credential registry inception (vcp) message event @@ -92,7 +91,7 @@ def incept( if toad != 0: # invalid toad raise ValueError("Invalid toad = {} for baks = {}".format(toad, baks)) - nonce = nonce if nonce is not None else coring.randomNonce() + nonce = nonce if nonce is not None else Salter().qb64 ked = dict(v=vs, # version string t=ilk, d="", @@ -106,7 +105,6 @@ def incept( ) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -119,7 +117,7 @@ def rotate( cuts=None, adds=None, version=Version, - kind=Serials.json, + kind=Kinds.json, ): """ Returns serder of registry rotation (brt) message event @@ -207,7 +205,6 @@ def rotate( ) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -215,7 +212,7 @@ def issue( vcdig, regk, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None ): """ Returns serder of issuance (iss) message event @@ -248,7 +245,6 @@ def issue( ked["dt"] = dt serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -257,7 +253,7 @@ def revoke( regk, dig, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None ): """ Returns serder of backerless credential revocation (rev) message event @@ -298,7 +294,6 @@ def revoke( _, ked = coring.Saider.saidify(sad=ked) serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -308,7 +303,7 @@ def backerIssue( regsn, regd, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None, ): """ Returns serder of backer issuance (bis) message event @@ -351,7 +346,6 @@ def backerIssue( ked["dt"] = dt serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -362,7 +356,7 @@ def backerRevoke( regd, dig, version=Version, - kind=Serials.json, + kind=Kinds.json, dt=None ): """ Returns serder of backer credential revocation (brv) message event @@ -406,7 +400,6 @@ def backerRevoke( ked["dt"] = dt serder = serdering.SerderKERI(sad=ked, makify=True) - serder._verify() # raises error if fails verifications return serder @@ -522,7 +515,7 @@ def vcstate(vcpre, ra=None, dts=None, # default current datetime version=Version, - kind=Serials.json, + kind=Kinds.json, ): """ Returns the credential transaction state notification @@ -592,7 +585,7 @@ def query(regk, dtb=None, stamp=None, version=Version, - kind=Serials.json + kind=Kinds.json ): """ Returns serder of credentialquery (qry) event message. @@ -644,7 +637,7 @@ class Tever: Has the following public attributes and properties: Class Attributes: - .NoBackers is Boolean + .NoRegistrarBackers is Boolean True means do not allow backers (default to witnesses of controlling KEL) False means allow backers (ignore witnesses of controlling KEL) @@ -666,7 +659,7 @@ class Tever: .noBackers is boolean trait True means do not allow backers """ - NoBackers = False + NoRegistrarBackers = False def __init__(self, cues=None, rsr=None, serder=None, seqner=None, saider=None, bigers=None, db=None, reger=None, noBackers=None, estOnly=None, @@ -718,12 +711,6 @@ def __init__(self, cues=None, rsr=None, serder=None, seqner=None, saider=None, raise ValidationError("Expected ilk {} got {} for evt: {}".format(Ilks.vcp, ilk, serder)) self.ilk = ilk - labels = VCP_LABELS - for k in labels: - if k not in serder.ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, serder.ked)) - self.incept(serder=serder) self.config(serder=serder, noBackers=noBackers, estOnly=estOnly) @@ -820,14 +807,14 @@ def incept(self, serder): """ ked = serder.ked - self.pre = ked["ii"] - self.prefixer = Prefixer(qb64=serder.pre) - if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix - raise ValidationError("Invalid prefix = {} for registry inception evt = {}." - .format(self.prefixer.qb64, ked)) + self.pre = ked["ii"] # which is not the AID of the serder in ked["i"] + self.prefixer = Prefixer(qb64=serder.pre) # this not related to self.pre + #if not self.prefixer.verify(ked=ked, prefixed=True): # invalid prefix + #raise ValidationError("Invalid prefix = {} for registry inception evt = {}." + #.format(self.prefixer.qb64, ked)) + - sn = ked["s"] - self.sn = validateSN(sn, inceptive=True) + self.sn = Number(numh=ked["s"]).validate(inceptive=True).sn self.cuts = [] # always empty at inception since no prev event self.adds = [] # always empty at inception since no prev event @@ -864,7 +851,7 @@ def config(self, serder, noBackers=None, estOnly=None): """ # assign traits self.noBackers = (True if (noBackers if noBackers is not None - else self.NoBackers) + else self.NoRegistrarBackers) else False) # ensure default noBackers is boolean self.estOnly = (True if (estOnly if estOnly is not None @@ -895,12 +882,13 @@ def update(self, serder, seqner=None, saider=None, bigers=None): ked = serder.ked ilk = ked["t"] - sn = ked["s"] + #sn = ked["s"] icp = ilk in (Ilks.iss, Ilks.bis) # validate SN for - sn = validateSN(sn, inceptive=icp) + #sn = validateSN(sn, inceptive=icp) + sn = Number(numh=ked["s"]).validate(inceptive=icp).sn if ilk in (Ilks.vrt,): if self.noBackers is True: @@ -961,8 +949,8 @@ def rotate(self, serder, sn): ilk = ked["t"] dig = ked["p"] - # XXXX should there be validation of labels here - labels = VRT_LABELS # assumes ilk == Ilks.vrt + + #labels = VRT_LABELS # assumes ilk == Ilks.vrt #for k in labels: #if k not in ked: #raise ValidationError("Missing element = {} from {} event for " @@ -1051,12 +1039,11 @@ def issue(self, serder, seqner, saider, sn, bigers=None): ilk = ked["t"] vci = vcpre - labels = ISS_LABELS if ilk == Ilks.iss else BIS_LABELS - - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) + #labels = ISS_LABELS if ilk == Ilks.iss else BIS_LABELS + #for k in labels: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event for " + #"evt = {}.".format(k, ilk, ked)) if ilk == Ilks.iss: # simple issue if self.noBackers is False: @@ -1118,12 +1105,11 @@ def revoke(self, serder, seqner, saider, sn, bigers=None): vcpre = ked["i"] ilk = ked["t"] - labels = REV_LABELS if ilk == Ilks.rev else BRV_LABELS - - for k in labels: - if k not in ked: - raise ValidationError("Missing element = {} from {} event for " - "evt = {}.".format(k, ilk, ked)) + #labels = REV_LABELS if ilk == Ilks.rev else BRV_LABELS + #for k in labels: + #if k not in ked: + #raise ValidationError("Missing element = {} from {} event for " + #"evt = {}.".format(k, ilk, ked)) # have to compare with VC issuance serder vci = vcpre @@ -1186,7 +1172,7 @@ def vcState(self, vci): status (Serder): transaction event state notification message """ digs = [] - for _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): + for _, _, dig in self.reger.getTelItemPreIter(pre=vci.encode("utf-8")): digs.append(dig) if len(digs) == 0: @@ -1267,8 +1253,8 @@ def logEvent(self, pre, sn, serder, seqner, saider, bigers=None, baks=None): self.reger.tets.pin(keys=(pre.decode("utf-8"), dig.decode("utf-8")), val=coring.Dater()) self.reger.putTvt(key, serder.raw) self.reger.putTel(snKey(pre, sn), dig) - logger.info("Tever: Added to TEL %s valid event=%s SAID=%s reg=%.8s... iss=%s", - serder.ilk, serder.pre, serder.said, self.regk, self.pre) + logger.info("Tever: Added to TEL valid %s event %s said=%s reg=%.8s iss=%.8s", + serder.ilk, pre.decode(), serder.said, self.regk, self.pre) logger.debug("TEL Event Body=\n%s\n", serder.pretty()) def valAnchorBigs(self, serder, seqner, saider, bigers, toad, baks): @@ -1320,11 +1306,10 @@ def valAnchorBigs(self, serder, seqner, saider, bigers, toad, baks): if len(bindices) < toad: # not fully witnessed yet self.escrowPWEvent(serder=serder, seqner=seqner, saider=saider, bigers=bigers) - msg = (f"Failure satisfying toad={toad} on witness sigs " - f"for {[siger.qb64 for siger in bigers]} " - f"for evt = {serder.said}.") - logger.info(msg) - logger.debug(f"Event Body=\n{serder.ked}\n") + msg = (f"Failure satisfying toad = {toad} on witness sigs " + f"for {[siger.qb64 for siger in bigers]} for evt = {serder.said}") + logger.info("Tever: %s", msg) + logger.debug(f"Event Body=\n%s\n", serder.pretty()) raise MissingWitnessSignatureError(msg) return bigers @@ -1397,8 +1382,8 @@ def escrowPWEvent(self, serder, seqner, saider, bigers=None): self.reger.putTibs(dgkey, [biger.qb64b for biger in bigers]) self.reger.putTvt(dgkey, serder.raw) self.reger.putTwe(snKey(serder.preb, serder.sn), serder.saidb) - logger.info("Tever state: Escrowed partially witnessed " - "event = %s\n", serder.ked) + logger.debug("Tever state: Escrowed partially witnessed " + "event = %s", serder.ked) def escrowALEvent(self, serder, seqner, saider, bigers=None, baks=None): """ Update associated logs for escrow of anchorless event @@ -1424,8 +1409,8 @@ def escrowALEvent(self, serder, seqner, saider, bigers=None, baks=None): self.reger.delBaks(key) self.reger.putBaks(key, [bak.encode("utf-8") for bak in baks]) self.reger.putTvt(key, serder.raw) - logger.info("Tever: Escrowed anchorless event event = %s", serder.said) - logger.debug("Event body=\n%s\n", serder.ked) + logger.debug("Tever state: Escrowed anchorless event " + "event = %s", serder.ked) return self.reger.putTae(snKey(serder.preb, serder.sn), serder.saidb) def getBackerState(self, ked): @@ -1544,13 +1529,14 @@ def processEvent(self, serder, seqner=None, saider=None, wigers=None): regk = self.registryKey(serder) pre = serder.pre ked = serder.ked - sn = ked["s"] + #sn = ked["s"] ilk = ked["t"] inceptive = ilk in (Ilks.vcp, Ilks.iss, Ilks.bis) # validate SN for - sn = validateSN(sn, inceptive=inceptive) + #sn = validateSN(sn, inceptive=inceptive) + sn = Number(numh=ked["s"]).validate(inceptive=inceptive).sn if not self.lax: if self.local: @@ -1559,9 +1545,8 @@ def processEvent(self, serder, seqner=None, saider=None, wigers=None): "".format(regk, self.registries)) else: if regk in self.registries: # local event when not in local mode - pass - # raise ValueError("Local event regk={} when nonlocal mode." - # "".format(regk)) + raise ValueError("Local event regk={} when nonlocal mode." + "".format(regk)) if regk not in self.tevers: # first seen for this registry if ilk in [Ilks.vcp]: @@ -1584,14 +1569,19 @@ def processEvent(self, serder, seqner=None, saider=None, wigers=None): else: # out of order, need to escrow self.escrowOOEvent(serder=serder, seqner=seqner, saider=saider) - raise OutOfOrderError("escrowed out of order event {}".format(ked)) + msg = f"Escrowed out of order event of type = {ilk} pre = {pre} SAID = {serder.said}" + logger.debug("Tevery: %s", msg) + logger.debug("TEL Event Body=\n%s\n", serder.pretty()) + raise OutOfOrderError(msg) else: if ilk in (Ilks.vcp,): # we don't have multiple signatures to verify so this - # is already first seen and then lifely duplicitious - logger.debug("Likely Duplicitous event Body=%s", serder.pretty()) - raise LikelyDuplicitousError(f"Likely Duplicitous event={serder.said}") + # is already first seen and then likely duplicitious + msg = f"Likely Duplicitous Event of type={serder.ilk} sn={sn} SAID={serder.said}" + logger.debug("Tevery: %s", msg) + logger.debug("TEL Event Body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) tever = self.tevers[regk] tever.cues = self.cues @@ -1617,8 +1607,10 @@ def processEvent(self, serder, seqner=None, saider=None, wigers=None): # self.cues.append(dict(kin="receipt", serder=serder)) pass else: # duplicitious - logger.debug("Likely Duplicitous event Body=%s", serder.pretty()) - raise LikelyDuplicitousError(f"Likely Duplicitous event={serder.said} with sn {serder.sn}") + msg = f"Likely Duplicitous Event type={serder.ilk} sn={sn} SAID={serder.said}" + logger.debug("Tevery: %s", msg) + logger.debug("TEL Event Body=\n%s\n", serder.pretty()) + raise LikelyDuplicitousError(msg) def processQuery(self, serder, source=None, sigers=None, cigars=None): """ Process TEL query event message (qry) @@ -2010,8 +2002,8 @@ def escrowOOEvent(self, serder, seqner, saider): sealet = seqner.qb64b + saider.qb64b self.reger.putAnc(key, sealet) self.reger.putOot(snKey(serder.preb, serder.sn), serder.saidb) - logger.info("Tever state: Escrowed our of order TEL event " - "event = %s\n", serder.ked) + logger.debug("Tever state: Escrowed our of order TEL event " + "event = %s", serder.ked) def processEscrows(self): """ Loop through escrows and process and events that may now be finalized """ @@ -2032,9 +2024,9 @@ def processEscrows(self): except Exception as ex: # log diagnostics errors etc if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery escrow process error: %s\n", ex.args[0]) + logger.exception("Tevery escrow process error: %s", ex.args[0]) else: - logger.error("Tevery escrow process error: %s\n", ex.args[0]) + logger.error("Tevery escrow process error: %s", ex.args[0]) def processEscrowOutOfOrders(self): """ Loop through out of order escrow: @@ -2047,32 +2039,29 @@ def processEscrowOutOfOrders(self): 5. Remove event digest from oots if processed successfully or a non-out-of-order event occurs. """ - for (pre, snb, digb) in self.reger.getOotItemIter(): + for key, digb in self.reger.getOotItemIter(): # (pre, snb, digb) in self.reger.getOotItemIter() try: - sn = int(snb, 16) + #sn = int(snb, 16) + pre, sn = splitSnKey(key) dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) if traw is None: # no event so raise ValidationError which unescrows below - logger.info("Tevery unescrow error: Missing event at." - "dig = %s\n", bytes(digb)) - - raise ValidationError("Missing escrowed evt at dig = {}." - "".format(bytes(digb))) + msg = f"OOO Missing escrowed event at dig = {bytes(digb).decode()}" + logger.info("Tevery unescrow error: %s", msg) + raise ValidationError(msg) tserder = serdering.SerderKERI(raw=bytes(traw)) # escrowed event bigers = None if tibs := self.reger.getTibs(key=dgkey): - bigers = [coring.Siger(qb64b=tib) for tib in tibs] + bigers = [indexing.Siger(qb64b=tib) for tib in tibs] couple = self.reger.getAnc(dgkey) if couple is None: - logger.info("Tevery unescrow error: Missing anchor at." - "dig = %s\n", bytes(digb)) - - raise ValidationError("Missing escrowed anchor at dig = {}." - "".format(bytes(digb))) + msg = f"OOO Missing escrowed anchor at dig = {bytes(digb).decode()}" + logger.info("Tevery unescrow error: %s", msg) + raise ValidationError(msg) ancb = bytearray(couple) seqner = coring.Seqner(qb64b=ancb, strip=True) saider = coring.Saider(qb64b=ancb, strip=True) @@ -2082,24 +2071,25 @@ def processEscrowOutOfOrders(self): except OutOfOrderError as ex: # still waiting on missing prior event to validate if logger.isEnabledFor(logging.TRACE): - logger.trace("Tevery unescrow failed: %s\n", ex.args[0]) - logger.exception("Tevery unescrow failed: %s\n", ex.args[0]) + logger.trace("Tevery: OOO unescrow failed: %s\n", ex.args[0]) + logger.exception("Tevery: OOO unescrow failed: %s\n", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.reger.delOot(snKey(pre, sn)) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery unescrowed: %s\n", ex.args[0]) + logger.exception("Tevery: OOO unescrowed: %s", ex.args[0]) else: - logger.error("Tevery unescrowed: %s\n", ex.args[0]) + logger.error("Tevery: OOO unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.reger.delOot(snKey(pre, sn)) # removes from escrow - logger.info("Tevery unescrow succeeded in valid event: event = %s", tserder.said) - logger.debug("Event Body=\n%s\n", tserder.pretty()) + logger.info("Tevery: OOO unescrow succeeded in valid event: " + "said=%s", tserder.said) + logger.debug("Event=\n%s\n", tserder.pretty()) def processEscrowAnchorless(self): """ Process escrow of TEL events received before the anchoring KEL event. @@ -2113,27 +2103,28 @@ def processEscrowAnchorless(self): 6. Remove event digest from oots if processed successfully or a non-anchorless event occurs. """ - for (pre, snb, digb) in self.reger.getTaeItemIter(): - sn = int(snb, 16) + for key, digb in self.reger.getTaeItemIter(): #(pre, snb, digb) in self.reger.getTaeItemIter() + pre, sn = splitSnKey(key) + #sn = int(snb, 16) try: dgkey = dgKey(pre, digb) traw = self.reger.getTvt(dgkey) if traw is None: # no event so raise ValidationError which unescrows below - msg = f"Tevery: anchorless escrow unescrow error: Missing event at dig = {bytes(digb)}" - logger.trace(msg) + msg = f"ANC Missing escrowed event at dig = {bytes(digb).decode()}" + logger.trace("Tevery unescrow error: %s", msg) raise ValidationError(msg) tserder = serdering.SerderKERI(raw=bytes(traw)) # escrowed event bigers = None if tibs := self.reger.getTibs(key=dgkey): - bigers = [coring.Siger(qb64b=tib) for tib in tibs] + bigers = [indexing.Siger(qb64b=tib) for tib in tibs] couple = self.reger.getAnc(dgkey) if couple is None: - msg = f"Tevery: anchorless escrow unescrow error: Missing anchor at dig = {bytes(digb)}" - logger.trace(msg) + msg = f"ANC Missing escrowed anchor at dig = {bytes(digb).decode()}" + logger.trace("Tevery unescrow error: %s", msg) raise MissingAnchorError(msg) ancb = bytearray(couple) seqner = coring.Seqner(qb64b=ancb, strip=True) @@ -2143,23 +2134,24 @@ def processEscrowAnchorless(self): except MissingAnchorError as ex: # still waiting on missing prior event to validate - if logger.isEnabledFor(logging.TRACE): - logger.trace("Tevery: anchorless escrow unescrow failed: %s\n", ex.args[0]) - logger.exception("Tevery: anchorless escrow unescrow failed: %s\n", ex.args[0]) + if logger.isEnabledFor(logging.DEBUG): + logger.exception("Tevery ANC unescrow failed: %s", ex.args[0]) + else: + logger.error("Tevery ANC unescrow failed: %s", ex.args[0]) except Exception as ex: # log diagnostics errors etc # error other than out of order so remove from OO escrow self.reger.delTae(snKey(pre, sn)) # removes one escrow at key val if logger.isEnabledFor(logging.DEBUG): - logger.exception("Tevery: anchorless escrow other error on unescrow: %s\n", ex.args[0]) + logger.exception("Tevery ANC unescrowed: %s", ex.args[0]) else: - logger.error("Tevery: anchorless escrow other error on unescrow: %s\n", ex.args[0]) + logger.error("Tevery ANC unescrowed: %s", ex.args[0]) else: # unescrow succeeded, remove from escrow # We don't remove all escrows at pre,sn because some might be # duplicitous so we process remaining escrows in spite of found # valid event escrow. self.reger.delTae(snKey(pre, sn)) # removes from escrow - logger.info("Tevery: anchorless escrow unescrow succeeded in valid event: " - "event = %s", tserder.said) - logger.debug("Event body=\n%s\n", tserder.pretty()) + logger.info("Tevery ANC unescrow succeeded in valid event: " + "said=%s", tserder.said) + logger.debug("event=\n%s\n", tserder.pretty()) diff --git a/src/keri/vdr/verifying.py b/src/keri/vdr/verifying.py index 4ebf9b420..139d94564 100644 --- a/src/keri/vdr/verifying.py +++ b/src/keri/vdr/verifying.py @@ -255,7 +255,7 @@ def _processEscrow(self, db, timeout, etype: Type[Exception]): if (dtnow - dte) > datetime.timedelta(seconds=timeout): # escrow stale so raise ValidationError which unescrows below logger.info("Verifier unescrow error: Stale event escrow " - " at said = %s\n", said) + " at said = %s", said) raise kering.ValidationError("Stale event escrow " "at said = {}.".format(said)) @@ -270,14 +270,14 @@ def _processEscrow(self, db, timeout, etype: Type[Exception]): # error other than missing sigs so remove from PA escrow db.rem(said) if logger.isEnabledFor(logging.DEBUG): - logger.exception("Verifier unescrowed: %s\n", ex.args[0]) + logger.exception("Verifier unescrowed: %s", ex.args[0]) else: - logger.error("Verifier unescrowed: %s\n", ex.args[0]) + logger.error("Verifier unescrowed: %s", ex.args[0]) else: db.rem(said) logger.info("Verifier unescrow succeeded in valid group op: " - "creder = %s", creder.said) - logger.debug("Creder body=\n%s\n", creder.pretty()) + "creder=%s", creder.said) + logger.debug(f"Event=\n%s\n", creder.pretty()) def saveCredential(self, creder, prefixer, seqner, saider): """ Write the credential and associated indicies to the database @@ -300,7 +300,7 @@ def saveCredential(self, creder, prefixer, seqner, saider): self.reger.issus.add(keys=issuer, val=saider) self.reger.schms.add(keys=schema, val=saider) - if 'i' in creder.attrib: + if not isinstance(creder.attrib, str) and 'i' in creder.attrib: subject = creder.attrib["i"].encode("utf-8") self.reger.subjs.add(keys=subject, val=saider) diff --git a/src/keri/vdr/viring.py b/src/keri/vdr/viring.py index 0c932e128..4a2cd4d8d 100644 --- a/src/keri/vdr/viring.py +++ b/src/keri/vdr/viring.py @@ -7,23 +7,22 @@ Provides public simple Verifiable Credential Issuance/Revocation Registry A special purpose Verifiable Data Registry (VDR) """ - +import os from dataclasses import dataclass, field, asdict -from ordered_set import OrderedSet as oset -from ..db import koming, subing, escrowing +from ordered_set import OrderedSet as oset -from .. import kering +from keri import help +from .. import kering, core from ..app import signing -from ..core import coring, serdering +from ..core import coring, serdering, indexing, counting from ..db import dbing, basing +from ..db import koming, subing, escrowing +from ..db.dbing import snKey from ..vdr import eventing -from keri import help - logger = help.ogler.getLogger() - class rbdict(dict): """ Reger backed read through cache for registry state @@ -171,6 +170,9 @@ def openReger(name="test", **kwa): """ return dbing.openLMDB(cls=Reger, name=name, **kwa) +# Env var for configuring LMDB size for the Keeper database +KERIRegerMapSizeKey = "KERI_REGER_MAP_SIZE" + class Reger(dbing.LMDBer): """ Reger sets up named sub databases for TEL registry @@ -277,6 +279,15 @@ def __init__(self, headDirPath=None, reopen=True, **kwa): else: self._tevers = dict() + mapSize = os.getenv(dbing.KERIRegerMapSizeKey) or os.getenv(dbing.KERILMDBMapSizeKey) + if mapSize is not None: + try: + self.MapSize = int(mapSize) + except ValueError: + logger.error(f"LMDB map size environment variable must be an integer value > 1! " + f"Use {dbing.KERIRegerMapSizeKey} or {dbing.KERILMDBMapSizeKey}") + raise + super(Reger, self).__init__(headDirPath=headDirPath, reopen=reopen, **kwa) @@ -329,7 +340,7 @@ def reopen(self, **kwa): # given by quintuple (saider.qb64, path, prefixer.qb64, seqner.q64, diger.qb64) # of credential and trans signer's key state est evt to val Siger for each # signature. - self.spsgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=coring.Siger) + self.spsgs = subing.CesrIoSetSuber(db=self, subkey='ssgs.', klas=indexing.Siger) # all sad path scgs (sad pathed non-indexed signature serializations) maps # couple (SAD SAID, path) to couple (Verfer, Cigar) of nontrans signer of signature in Cigar @@ -385,40 +396,6 @@ def reopen(self, **kwa): return self.env - def clearEscrows(self): - """Clear credential event escrows""" - # self.oots, self.twes, self.taes - count = 0 - for (k, _) in self.getOotItemIter(): - count += 1 - self.delOot(k) - logger.info(f"TEL: Cleared {count} out of order escrows.") - - count = 0 - for (k, ) in self.getAllItemIter(self.twes): - count += 1 - self.delTwe(k) - logger.info(f"TEL: Cleared {count} partially witnessed escrows.") - - count = 0 - for (k, _) in self.getAllItemIter(self.taes): - count += 1 - self.delTae(k) - logger.info(f"TEL: Cleared {count} anchorless escrows.") - - for name, sub, desc in [ - ( 'mre', self.mre, 'missing registry escrows'), - ( 'mce', self.mce, 'broken chain escrows'), - ( 'mse', self.mse, 'missing schema escrows'), - ('cmse', self.cmse, 'missing signature escrows'), - ('tpwe', self.tpwe, 'partial witness escrows'), - ('tmse', self.tmse, 'multisig escrows'), - ('tede', self.tede, 'event dissemination escrows') - ]: - sub.trim() - logger.info(f"TEL: Cleared escrow ({name.ljust(5)}): {desc}") - logger.info("Cleared TEL escrows") - def cloneCreds(self, saids, db): """ Returns fully expanded credential with chained credentials attached. @@ -437,11 +414,7 @@ def cloneCreds(self, saids, db): atc = bytearray(signing.serialize(creder, prefixer, seqner, saider)) del atc[0:creder.size] - regk = creder.regi - status = self.tevers[regk].vcState(saider.qb64) - schemer = db.schema.get(creder.schema) - - iss = bytearray(self.cloneTvtAt(creder.said, sn=0 if status.et == coring.Ilks.iss else 1)) + iss = bytearray(self.cloneTvtAt(creder.said)) iserder = serdering.SerderKERI(raw=iss) issatc = bytes(iss[iserder.size:]) @@ -458,6 +431,10 @@ def cloneCreds(self, saids, db): chainSaids.append(coring.Saider(qb64=p["n"])) chains = self.cloneCreds(chainSaids, db) + regk = creder.regi + status = self.tevers[regk].vcState(saider.qb64) + schemer = db.schema.get(creder.schema) + cred = dict( sad=creder.sad, atc=atc.decode("utf-8"), @@ -474,11 +451,11 @@ def cloneCreds(self, saids, db): ) ) - ctr = coring.Counter(qb64b=iss, strip=True) - if ctr.code == coring.CtrDex.AttachedMaterialQuadlets: - ctr = coring.Counter(qb64b=iss, strip=True) + ctr = core.Counter(qb64b=iss, strip=True, gvrsn=kering.Vrsn_1_0) + if ctr.code == counting.CtrDex_1_0.AttachmentGroup: + ctr = core.Counter(qb64b=iss, strip=True, gvrsn=kering.Vrsn_1_0) - if ctr.code == coring.CtrDex.SealSourceCouples: + if ctr.code == counting.CtrDex_1_0.SealSourceCouples: coring.Seqner(qb64b=iss, strip=True) saider = coring.Saider(qb64b=iss) @@ -520,6 +497,8 @@ def cloneCred(self, said): """ creder = self.creds.get(keys=(said,)) + if creder is None: + raise kering.MissingEntryError(f"no credential found with said {said}") prefixer, seqner, saider = self.cancs.get(keys=(said,)) return creder, prefixer, seqner, saider @@ -541,7 +520,7 @@ def clonePreIter(self, pre, fn=0): if hasattr(pre, 'encode'): pre = pre.encode("utf-8") - for fn, dig in self.getTelItemPreIter(pre, fn=fn): + for _, fn, dig in self.getTelItemPreIter(pre, fn=fn): msg = self.cloneTvt(pre, dig) yield msg @@ -560,24 +539,24 @@ def cloneTvt(self, pre, dig): # add indexed backer signatures to attachments if tibs := self.getTibs(key=dgkey): - atc.extend(coring.Counter(code=coring.CtrDex.WitnessIdxSigs, - count=len(tibs)).qb64b) + atc.extend(core.Counter(core.Codens.WitnessIdxSigs, count=len(tibs), + gvrsn=kering.Vrsn_1_0).qb64b) for tib in tibs: atc.extend(tib) # add authorizer (delegator/issure) source seal event couple to attachments couple = self.getAnc(dgkey) if couple is not None: - atc.extend(coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1).qb64b) + atc.extend(core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(couple) # prepend pipelining counter to attachments if len(atc) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(atc))) - pcnt = coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, - count=(len(atc) // 4)).qb64b + pcnt = core.Counter(core.Codens.AttachmentGroup, count=(len(atc) // 4), + gvrsn=kering.Vrsn_1_0).qb64b msg.extend(pcnt) msg.extend(atc) return msg @@ -608,7 +587,8 @@ def sources(self, db, creder): for said in saids: screder, prefixer, seqner, saider = self.cloneCred(said=said) - atc = bytearray(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + atc = bytearray(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(prefixer.qb64b) atc.extend(seqner.qb64b) atc.extend(saider.qb64b) @@ -703,7 +683,7 @@ def getTelItemPreIter(self, pre, fn=0): pre is bytes of itdentifier prefix fn is int fn to resume replay. Earliset is fn=0 """ - return self.getAllOrdItemPreIter(db=self.tels, pre=pre, on=fn) + return self.getOnItemIter(db=self.tels, key=pre, on=fn) def cntTels(self, pre, fn=0): """ @@ -717,7 +697,7 @@ def cntTels(self, pre, fn=0): if hasattr(pre, "encode"): pre = pre.encode("utf-8") # convert str to bytes - return self.cntValsAllPre(db=self.tels, pre=pre, on=fn) + return self.cntOnVals(db=self.tels, key=pre, on=fn) def getTibs(self, key): """ @@ -809,6 +789,13 @@ def delTwe(self, key): """ return self.delVal(self.twes, key) + def getTweItemIter(self): + """ + Return iterator of all items in .twes + + """ + return self.getTopItemIter(self.twes) + def putTae(self, key, val): """ Use snKey() @@ -841,7 +828,8 @@ def getTaeItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.taes, split=True) + return self.getTopItemIter(self.taes) + def delTae(self, key): """ @@ -884,7 +872,8 @@ def getOotItemIter(self): Return iterator of all items in .taes """ - return self.getAllItemIter(self.oots, split=True) + return self.getTopItemIter(self.oots) + def delOot(self, key): """ @@ -939,7 +928,7 @@ def putBaks(self, key, vals): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.putIoVals(self.baks, key, vals) + return self.putIoDupVals(self.baks, key, vals) def addBak(self, key, val): @@ -950,7 +939,7 @@ def addBak(self, key, val): Returns True If at least one of vals is added as dup, False otherwise Duplicates are inserted in insertion order. """ - return self.addIoVal(self.baks, key, val) + return self.addIoDupVal(self.baks, key, val) def getBaks(self, key): @@ -960,7 +949,7 @@ def getBaks(self, key): Returns empty list if no entry at key Duplicates are retrieved in insertion order. """ - return self.getIoVals(self.baks, key) + return self.getIoDupVals(self.baks, key) def getBaksIter(self, key): @@ -970,7 +959,7 @@ def getBaksIter(self, key): Raises StopIteration Error when empty Duplicates are retrieved in insertion order. """ - return self.getIoValsIter(self.baks, key) + return self.getIoDupValsIter(self.baks, key) def cntBaks(self, key): """ @@ -978,7 +967,7 @@ def cntBaks(self, key): Return count of backer prefixes at key Returns zero if no entry at key """ - return self.cntIoVals(self.baks, key) + return self.cntIoDupVals(self.baks, key) def delBaks(self, key): @@ -987,7 +976,7 @@ def delBaks(self, key): Deletes all values at key in db. Returns True If key exists in database Else False """ - return self.delIoVals(self.baks, key) + return self.delIoDupVals(self.baks, key) def delBak(self, key, val): @@ -1000,7 +989,50 @@ def delBak(self, key, val): key is bytes of key within sub db's keyspace val is dup val (does not include insertion ordering proem) """ - return self.delIoVal(self.baks, key, val) + return self.delIoDupVal(self.baks, key, val) + + def clearEscrows(self): + """Clear credential event escrows""" + # self.oots, self.twes, self.taes + count = 0 + for (k , _) in self.getOotItemIter(): + count += 1 + self.delOot(k) + logger.info(f"TEL: Cleared {count} out of order escrows.") + + count = 0 + for (k, _) in self.getTweItemIter(): + count += 1 + self.delTwe(k) + logger.info(f"TEL: Cleared {count} partially witnessed escrows.") + + count = 0 + for (k, _) in self.getTaeItemIter(): + count += 1 + self.delTae(k) + logger.info(f"TEL: Cleared {count} anchorless escrows.") + + for name, sub, desc in [ + ( 'mre', self.mre, 'missing registry escrows'), + ( 'mce', self.mce, 'broken chain escrows'), + ( 'mse', self.mse, 'missing schema escrows'), + ('cmse', self.cmse, 'missing signature escrows'), + ('tpwe', self.tpwe, 'partial witness escrows'), + ('tmse', self.tmse, 'multisig escrows'), + ('tede', self.tede, 'event dissemination escrows') + ]: + sub.trim() + logger.info(f"TEL: Cleared escrow ({name.ljust(5)}): {desc}") + + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + count = 0 + for keys, saider in self.txnsb.escrowdb.getItemIter(keys=(typ, "")): + count += 1 + self.txnsb.escrowdb.rem(keys=keys, val=saider) + self.txnsb.removeState(saider) + logger.info(f"TEL: Cleared {count} broker escrows ({typ})") + + logger.info("Cleared TEL escrows") def buildProof(prefixer, seqner, diger, sigers): @@ -1016,12 +1048,14 @@ def buildProof(prefixer, seqner, diger, sigers): """ prf = bytearray() - prf.extend(coring.Counter(coring.CtrDex.TransIdxSigGroups, count=1).qb64b) + prf.extend(core.Counter(core.Codens.TransIdxSigGroups, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) prf.extend(prefixer.qb64b) prf.extend(seqner.qb64b) prf.extend(diger.qb64b) - prf.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) + prf.extend(core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0).qb64b) for siger in sigers: prf.extend(siger.qb64b) @@ -1044,8 +1078,8 @@ def messagize(creder, proof): if len(proof) % 4: raise ValueError("Invalid attachments size={}, nonintegral" " quadlets.".format(len(proof))) - craw.extend(coring.Counter(code=coring.CtrDex.AttachedMaterialQuadlets, - count=(len(proof) // 4)).qb64b) + craw.extend(core.Counter(core.Codens.AttachmentGroup, count=(len(proof) // 4), + gvrsn=kering.Vrsn_1_0).qb64b) craw.extend(proof) return craw diff --git a/tests/app/__init__.py b/tests/app/__init__.py index 1264abdb8..312d14603 100644 --- a/tests/app/__init__.py +++ b/tests/app/__init__.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from keri.app import habbing +from keri import kering, core from keri.core import coring, eventing, parsing from keri.db import dbing @@ -54,13 +55,13 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): sigs.extend(bytes(hab3.db.getSigs(dgkey)[0])) evt = bytearray(eraw) - evt.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=3).qb64b) # attach cnt + evt.extend(core.Counter(code=core.Codens.ControllerIdxSigs, + count=3, gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt evt.extend(sigs) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev3) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev2) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev3, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) assert ghab1.pre in kev1.kevers assert ghab1.pre in kev2.kevers diff --git a/src/keri/app/kiwiing.py b/tests/app/cli/commands/contacts/__init__.py similarity index 100% rename from src/keri/app/kiwiing.py rename to tests/app/cli/commands/contacts/__init__.py diff --git a/tests/app/cli/commands/contacts/test_contacts.py b/tests/app/cli/commands/contacts/test_contacts.py new file mode 100644 index 000000000..1f3ba9724 --- /dev/null +++ b/tests/app/cli/commands/contacts/test_contacts.py @@ -0,0 +1,365 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.cli.commands.contacts.test_contacts module + +Tests for KLI contacts commands: get, add, rename, delete +""" +import json +import pytest + +import multicommand + +from keri.app import habbing, connecting +from keri.app.cli import commands + + +def test_contacts_get_by_aid(capsys): + """Test getting a contact by AID""" + joe = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + joed = dict(first="Joe", last="Jury", alias="joe", company="HCF") + + with habbing.openHby(name="test-get-aid", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=joe, data=joed) + + # Test the get logic directly + contact = org.get(joe) + assert contact is not None + assert contact["id"] == joe + assert contact["first"] == "Joe" + assert contact["alias"] == "joe" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "get", "--name", "test", "--aid", joe]) + assert args.handler is not None + assert args.aid == joe + assert args.alias is None + + +def test_contacts_get_by_alias(capsys): + """Test getting a contact by alias""" + bob = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + bobd = dict(first="Bob", last="Burns", alias="bob", company="HCF") + + with habbing.openHby(name="test-get-alias", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=bob, data=bobd) + + # Test the find/get logic directly + contacts = org.find('alias', f"^bob$") + assert len(contacts) == 1 + contact = contacts[0] + assert contact["id"] == bob + assert contact["first"] == "Bob" + assert contact["alias"] == "bob" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "get", "--name", "test", "--alias", "bob"]) + assert args.handler is not None + assert args.alias == "bob" + assert args.aid is None + + +def test_contacts_get_not_found(): + """Test getting a non-existent contact""" + with habbing.openHby(name="test-get-notfound", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + contact = org.get("ENonExistent123456789012345678901234567890123") + assert contact is None + + +def test_contacts_add_new(): + """Test adding a new contact""" + ken = "EFC7f_MEPE5dboc_E4yG15fnpMD34YaU3ue6vnDLodJU" + + with habbing.openHby(name="test-add-new", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + # Test the add logic directly (using update) + data = {'alias': 'ken', 'company': 'GLEIF', 'city': 'Frankfurt'} + org.update(ken, data) + + contact = org.get(ken) + assert contact["id"] == ken + assert contact["alias"] == "ken" + assert contact["company"] == "GLEIF" + assert contact["city"] == "Frankfurt" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "add", "--name", "test", + "--oobi", "http://127.0.0.1:5642/oobi/" + ken, + "--alias", "ken", "--field", "company=GLEIF", + "--field", "city=Frankfurt"]) + assert args.handler is not None + assert args.oobi == "http://127.0.0.1:5642/oobi/" + ken + assert args.alias == "ken" + assert args.fields == ["company=GLEIF", "city=Frankfurt"] + + +def test_contacts_add_update(): + """Test updating an existing contact""" + jen = "ED61oxVwVNf_olqR5wAhAjvuK59xuBOJXnJPGhwWDYoc" + jend = dict(first="Jen", last="Jones", alias="jen", company="GLEIF") + + with habbing.openHby(name="test-add-update", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=jen, data=jend) + + # Test the update logic - preserves existing fields + org.update(jen, {'mobile': '555-1212'}) + + contact = org.get(jen) + assert contact["id"] == jen + assert contact["first"] == "Jen" # preserved + assert contact["alias"] == "jen" # preserved + assert contact["mobile"] == "555-1212" # added + + +def test_contacts_add_field_parsing(): + """Test add command field parsing""" + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "add", "--name", "test", + "--oobi", "http://127.0.0.1:5642/oobi/Etest123", + "--field", "key1=value1", + "--field", "key2=value with spaces"]) + assert args.oobi == "http://127.0.0.1:5642/oobi/Etest123" + assert args.fields == ["key1=value1", "key2=value with spaces"] + + # Parse fields like the command does + data = {} + for field in args.fields: + key, val = field.split('=', 1) + data[key] = val + + assert data == {"key1": "value1", "key2": "value with spaces"} + + +def test_contacts_rename(): + """Test renaming a contact alias""" + sal = "Eo60ITGA69z4jNBU4RsvbgsjfAHFcTM2HVEXea1SvnXk" + sald = dict(first="Sally", last="Smith", alias="sally", company="GLEIF") + + with habbing.openHby(name="test-rename", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=sal, data=sald) + + # Test the rename logic directly + org.set(sal, 'alias', 'sal') + + contact = org.get(sal) + assert contact["alias"] == "sal" + + # Verify command parser is correct + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "rename", "--name", "test", + "--old-alias", "sally", "--alias", "sal"]) + assert args.handler is not None + assert args.old_alias == "sally" + assert args.alias == "sal" + + +def test_contacts_rename_by_aid(): + """Test renaming a contact by AID""" + joe = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + joed = dict(first="Joe", alias="joe") + + with habbing.openHby(name="test-rename-aid", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=joe, data=joed) + + org.set(joe, 'alias', 'joseph') + + contact = org.get(joe) + assert contact["alias"] == "joseph" + + # Verify command parser + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "rename", "--name", "test", + "--aid", joe, "--alias", "joseph"]) + assert args.aid == joe + assert args.alias == "joseph" + + +def test_contacts_rename_not_found(): + """Test finding a non-existent contact for rename""" + with habbing.openHby(name="test-rename-notfound", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + contacts = org.find('alias', f"^nonexistent$") + assert len(contacts) == 0 + + +def test_contacts_delete(): + """Test deleting a contact""" + bob = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + bobd = dict(first="Bob", last="Burns", alias="bob", company="HCF") + + with habbing.openHby(name="test-delete", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=bob, data=bobd) + + # Verify contact exists + assert org.get(bob) is not None + + # Test the delete logic directly + org.rem(bob) + + # Verify contact was deleted + assert org.get(bob) is None + + +def test_contacts_delete_parser(): + """Test delete command parser""" + parser = multicommand.create_parser(commands) + + # Test with --alias and --yes + args = parser.parse_args(["contacts", "delete", "--name", "test", + "--alias", "bob", "--yes"]) + assert args.handler is not None + assert args.alias == "bob" + assert args.yes is True + + # Test with --aid + args = parser.parse_args(["contacts", "delete", "--name", "test", + "--aid", "Etest123", "-y"]) + assert args.aid == "Etest123" + assert args.yes is True + + +def test_contacts_delete_not_found(): + """Test deleting a non-existent contact""" + with habbing.openHby(name="test-delete-notfound", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + contacts = org.find('alias', f"^nonexistent$") + assert len(contacts) == 0 + + contact = org.get("ENonExistent123456789012345678901234567890123") + assert contact is None + + +def test_contacts_list_parser(): + """Test list command parser (existing command)""" + parser = multicommand.create_parser(commands) + args = parser.parse_args(["contacts", "list", "--name", "test"]) + assert args.handler is not None + assert args.name == "test" + + +def test_contacts_query_parser(): + """Test query command parser""" + parser = multicommand.create_parser(commands) + + # Test with --contact-alias + args = parser.parse_args(["contacts", "query", "--name", "test", + "--alias", "myid", "--contact-alias", "mycontact"]) + assert args.handler is not None + assert args.alias == "myid" + assert args.contact_alias == "mycontact" + + # Test with --contact-aid + args = parser.parse_args(["contacts", "query", "--name", "test", + "--alias", "myid", "--contact-aid", "Etest123"]) + assert args.contact_aid == "Etest123" + + +def test_contacts_workflow(): + """Test complete contact workflow: add, rename, get, delete""" + wil = "EPzeu5_C80nzPc_BGUHVBkXXfNmlS55Ayl7Rd1I0gWFE" + + with habbing.openHby(name="test-workflow", temp=True) as hby: + org = connecting.Organizer(hby=hby) + + # 1. Add contact + org.update(wil, {'alias': 'will', 'first': 'Will', 'company': 'GLEIF'}) + contact = org.get(wil) + assert contact["alias"] == "will" + + # 2. Rename contact + org.set(wil, 'alias', 'william') + contact = org.get(wil) + assert contact["alias"] == "william" + + # 3. Find by new alias + contacts = org.find('alias', f"^william$") + assert len(contacts) == 1 + assert contacts[0]["first"] == "Will" + + # 4. Update with more fields + org.update(wil, {'mobile': '555-1234'}) + contact = org.get(wil) + assert contact["mobile"] == "555-1234" + assert contact["first"] == "Will" # preserved + + # 5. Delete contact + org.rem(wil) + assert org.get(wil) is None + + +def test_contacts_find_parser(): + """Test find command parser""" + parser = multicommand.create_parser(commands) + + # Default field is alias + args = parser.parse_args(["contacts", "find", "--name", "test", "--value", "cfca"]) + assert args.handler is not None + assert args.field == "alias" + assert args.value == "cfca" + + # Explicit field + args = parser.parse_args(["contacts", "find", "--name", "test", + "--field", "company", "--value", "GLEIF"]) + assert args.field == "company" + assert args.value == "GLEIF" + + +def test_contacts_find(): + """Test finding contacts by field value pattern""" + aid1 = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + aid2 = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + aid3 = "EFC7f_MEPE5dboc_E4yG15fnpMD34YaU3ue6vnDLodJU" + + with habbing.openHby(name="test-find", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=aid1, data={'alias': 'yuan.cfca', 'company': 'CFCA'}) + org.replace(pre=aid2, data={'alias': 'li.cfca', 'company': 'CFCA'}) + org.replace(pre=aid3, data={'alias': 'bob.gleif', 'company': 'GLEIF'}) + + # Find by alias pattern + contacts = org.find('alias', 'cfca') + assert len(contacts) == 2 + aliases = {c['alias'] for c in contacts} + assert aliases == {'yuan.cfca', 'li.cfca'} + + # Find by company field + contacts = org.find('company', 'GLEIF') + assert len(contacts) == 1 + assert contacts[0]['alias'] == 'bob.gleif' + + # No matches + contacts = org.find('alias', 'nonexistent') + assert len(contacts) == 0 + + +def test_contacts_multiple_alias_match(): + """Test handling of multiple contacts with similar aliases""" + joe1 = "EtyPSuUjLyLdXAtGMrsTt0-ELyWeU8fJcymHiGOfuaSA" + joe2 = "EuEQX8At31X96iDVpigv-rTdOKvFiWFunbJ1aDfq89IQ" + + with habbing.openHby(name="test-multi", temp=True) as hby: + org = connecting.Organizer(hby=hby) + org.replace(pre=joe1, data={'alias': 'joe'}) + org.replace(pre=joe2, data={'alias': 'joey'}) + + # Exact match should return one + contacts = org.find('alias', f"^joe$") + assert len(contacts) == 1 + assert contacts[0]['id'] == joe1 + + # Partial match returns both + contacts = org.find('alias', 'joe') + assert len(contacts) == 2 diff --git a/tests/app/cli/test_kli_commands.py b/tests/app/cli/test_kli_commands.py index afb021102..e473ae2ad 100644 --- a/tests/app/cli/test_kli_commands.py +++ b/tests/app/cli/test_kli_commands.py @@ -1,13 +1,20 @@ +import json import os import multicommand import pytest -from keri.app import directing, habbing + +from keri.kering import ValidationError + +from keri import core +from keri.core import coring + +from keri.app import directing + from keri.app.cli import commands from keri.app.cli.common import existing -from keri.core import coring -from keri.kering import ValidationError + TEST_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -17,7 +24,7 @@ def test_standalone_kli_commands(helpers, capsys): assert os.path.isdir("/usr/local/var/keri/ks/test") is False parser = multicommand.create_parser(commands) - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 args = parser.parse_args(["init", "--name", "test", "--nopasscode", "--salt", salt]) assert args.handler is not None doers = args.handler(args) @@ -238,35 +245,44 @@ def test_standalone_kli_commands(helpers, capsys): doers = args.handler(args) directing.runController(doers=doers) capesc = capsys.readouterr() - assert capesc.out == ('{\n' - ' "unverified-receipts": 0,\n' - ' "verified-receipts": 0,\n' - ' "partially-signed-events": [],\n' - ' "partially-witnessed-events": [],\n' - ' "unverified-event-indexed-couples": 0,\n' - ' "out-of-order-events": [],\n' - ' "likely-duplicitous-events": [],\n' - ' "query-not-found": 0,\n' - ' "partially-delegated-events": 0,\n' - ' "reply": 0,\n' - ' "failed-oobi": 0,\n' - ' "group-partial-witness": 0,\n' - ' "group-delegate": 0,\n' - ' "delegated-partial-witness": 0,\n' - ' "group-partial-signed": 0,\n' - ' "exchange-partial-signed": 0,\n' - ' "delegated-unanchored": 0,\n' - ' "tel-out-of-order": 0,\n' - ' "tel-partially-witnessed": 0,\n' - ' "tel-anchorless": 0,\n' - ' "missing-registry-escrow": [],\n' - ' "broken-chain-escrow": [],\n' - ' "missing-schema-escrow": [],\n' - ' "tel-missing-signature": 0,\n' - ' "tel-partial-witness-escrow": 0,\n' - ' "tel-multisig": 0,\n' - ' "tel-event-dissemination": 0\n' - '}\n') + escrows = ("""{ + "unverified-receipts": 0, + "verified-receipts": 0, + "out-of-order-events": [], + "partially-witnessed-events": [], + "partially-signed-events": [], + "likely-duplicitous-events": [], + "unverified-event-indexed-couples": 0, + "query-not-found": 0, + "partially-delegated-events": 0, + "reply": 0, + "failed-oobi": 0, + "group-partial-witness": 0, + "group-delegate": 0, + "delegated-partial-witness": 0, + "group-partial-signed": 0, + "exchange-partial-signed": 0, + "delegated-unanchored": 0, + "tel-out-of-order": 0, + "tel-partially-witnessed": 0, + "tel-anchorless": 0, + "missing-registry-escrow": [], + "broken-chain-escrow": [], + "missing-schema-escrow": [], + "tel-missing-signature": 0, + "tel-partial-witness-escrow": 0, + "tel-multisig": 0, + "tel-event-dissemination": 0, + "registry-missing-anchor": 0, + "registry-out-of-order": 0, + "credential-missing-registry": 0, + "credential-missing-anchor": 0, + "credential-out-of-order": 0 + } + """) + assert json.loads(capesc.out) == json.loads(escrows) + + def test_incept_and_rotate_opts(helpers, capsys): @@ -277,7 +293,7 @@ def test_incept_and_rotate_opts(helpers, capsys): assert os.path.isdir("/usr/local/var/keri/ks/test-opts") is False parser = multicommand.create_parser(commands) - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 args = parser.parse_args(["init", "--name", "test-opts", "--nopasscode", "--salt", salt]) assert args.handler is not None doers = args.handler(args) diff --git a/tests/app/test_agenting.py b/tests/app/test_agenting.py index 244549fd3..74206759e 100644 --- a/tests/app/test_agenting.py +++ b/tests/app/test_agenting.py @@ -7,20 +7,20 @@ from hio.base import doing, tyming -from keri import kering +from keri import kering, core from keri.core import coring, serdering -from keri.core.coring import Counter, CtrDex, Seqner +from keri.core.coring import Seqner from keri.help import nowIso8601 from keri.app import habbing, indirecting, agenting, directing -from keri.db import dbing +from keri.db import basing, dbing from keri.vdr import eventing, viring -def test_withness_receiptor(seeder): - with habbing.openHby(name="wan", salt=coring.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ - habbing.openHby(name="wil", salt=coring.Salter(raw=b'will-the-witness').qb64) as wilHby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby: +def test_witness_receiptor(seeder): + with habbing.openHby(name="wan", salt=core.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ + habbing.openHby(name="wil", salt=core.Salter(raw=b'will-the-witness').qb64) as wilHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: wanDoers = indirecting.setupWitness(alias="wan", hby=wanHby, tcpPort=5632, httpPort=5642) wilDoers = indirecting.setupWitness(alias="wil", hby=wilHby, tcpPort=5633, httpPort=5643) @@ -60,7 +60,7 @@ def __init__(self, hby, wanHab, wilHab, wesHab): super(ReceiptDoer, self).__init__(doers=[doing.doify(self.testDo)]) - def testDo(self, tymth, tock=0.0): + def testDo(self, tymth, tock=0.0, **kwa): """ Execute a series of kli commands for this test scenario """ # enter context self.wind(tymth) @@ -113,14 +113,14 @@ def testDo(self, tymth, tock=0.0): def test_witness_sender(seeder): - with habbing.openHby(name="wan", salt=coring.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ - habbing.openHby(name="wil", salt=coring.Salter(raw=b'will-the-witness').qb64) as wilHby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby: + with habbing.openHby(name="wan", salt=core.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ + habbing.openHby(name="wil", salt=core.Salter(raw=b'will-the-witness').qb64) as wilHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: # looks like bad magic value in seeder is causing this to fail pdoer = PublishDoer(wanHby, wilHby, wesHby, palHby, seeder) - directing.runController(doers=[pdoer], expire=15.0) + directing.runController(doers=[pdoer], expire=10.0) assert pdoer.done is True @@ -130,6 +130,12 @@ def __init__(self, wanHby, wilHby, wesHby, palHby, seeder): wanDoers = indirecting.setupWitness(alias="wan", hby=wanHby, tcpPort=5632, httpPort=5642) wilDoers = indirecting.setupWitness(alias="wil", hby=wilHby, tcpPort=5633, httpPort=5643) wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) + # Pull the regers out of the Doers so the regers are reused and do not trigger an LMDB error on reuse + self.regers = dict( + wan=next(doer.baser for doer in wanDoers if isinstance(doer, basing.BaserDoer)), + wil=next(doer.baser for doer in wilDoers if isinstance(doer, basing.BaserDoer)), + wes=next(doer.baser for doer in wesDoers if isinstance(doer, basing.BaserDoer)), + ) wanHab = wanHby.habByName(name="wan") wilHab = wilHby.habByName(name="wil") @@ -145,7 +151,7 @@ def __init__(self, wanHby, wilHby, wesHby, palHby, seeder): super(PublishDoer, self).__init__(doers=doers) - def testDo(self, tymth, tock=0.0): + def testDo(self, tymth, tock=0.0, **kwa): """ Run the test and exit and remove all child doers when done """ self.wind(tymth) self.tock = tock @@ -155,7 +161,8 @@ def testDo(self, tymth, tock=0.0): serder = eventing.issue(vcdig=regser.pre, regk="EbA1o_bItVC9i6YB3hr2C3I_Gtqvz02vCmavJNoBA3Jg") msg = bytearray(serder.raw) - msg.extend(Counter(CtrDex.SealSourceCouples, count=1).qb64b) + msg.extend(core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(Seqner(sn=self.palHab.kever.sn).qb64b) msg.extend(self.palHab.kever.serder.saidb) @@ -169,7 +176,7 @@ def testDo(self, tymth, tock=0.0): assert cue["msg"] == msg for name in ["wes", "wil", "wan"]: - reger = viring.Reger(name=name) + reger = self.regers[name] while True: raw = reger.getTvt(dbing.dgKey(serder.preb, serder.saidb)) if raw: @@ -183,11 +190,11 @@ def testDo(self, tymth, tock=0.0): def test_witness_inquisitor(mockHelpingNowUTC, seeder): - with habbing.openHby(name="wan", salt=coring.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ - habbing.openHby(name="wil", salt=coring.Salter(raw=b'will-the-witness').qb64) as wilHby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ - habbing.openHby(name="qin", salt=coring.Salter(raw=b'abcdef0123456789').qb64) as qinHby: + with habbing.openHby(name="wan", salt=core.Salter(raw=b'wann-the-witness').qb64) as wanHby, \ + habbing.openHby(name="wil", salt=core.Salter(raw=b'will-the-witness').qb64) as wilHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ + habbing.openHby(name="qin", salt=core.Salter(raw=b'abcdef0123456789').qb64) as qinHby: wanDoers = indirecting.setupWitness(alias="wan", hby=wanHby, tcpPort=5632, httpPort=5642) wilDoers = indirecting.setupWitness(alias="wil", hby=wilHby, tcpPort=5633, httpPort=5643) wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) diff --git a/tests/app/test_connecting.py b/tests/app/test_connecting.py index 278efa891..448cbc37a 100644 --- a/tests/app/test_connecting.py +++ b/tests/app/test_connecting.py @@ -9,7 +9,7 @@ from keri import kering from keri.app import connecting, habbing -from keri.core import coring +from keri.core import signing def test_organizer(): @@ -144,7 +144,7 @@ def test_organizer(): 'zip': '08807'} # Update the Jen's data signature by signing garbage - nonce = coring.randomNonce() + nonce = signing.Salter().qb64 cigar = hby.signator.sign(ser=nonce.encode("utf-8")) hby.db.ccigs.pin(keys=(jen,), val=cigar) with pytest.raises(kering.ValidationError): diff --git a/tests/app/test_delegating.py b/tests/app/test_delegating.py index f72659036..f5c83870f 100644 --- a/tests/app/test_delegating.py +++ b/tests/app/test_delegating.py @@ -7,19 +7,23 @@ from hio.base import doing, tyming from keri import kering -from keri.app import habbing, delegating, indirecting, agenting, notifying + +from keri import core from keri.core import eventing, parsing, coring + +from keri.app import habbing, delegating, indirecting, agenting, notifying + from keri.db import dbing -def test_boatswain(seeder): - with habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ - habbing.openHby(name="del", salt=coring.Salter(raw=b'0123456789ghijkl').qb64) as delHby: +def test_anchorer(seeder): + with habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby, \ + habbing.openHby(name="del", salt=core.Salter(raw=b'0123456789ghijkl').qb64) as delHby: wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) witDoer = agenting.Receiptor(hby=palHby) - bts = delegating.Sealer(hby=delHby) + bts = delegating.Anchorer(hby=delHby) wesHab = wesHby.habByName(name="wes") seeder.seedWitEnds(palHby.db, witHabs=[wesHab], protocols=[kering.Schemes.http]) @@ -33,7 +37,7 @@ def test_boatswain(seeder): bts=bts ) - doers = wesDoers + [witDoer, bts, doing.doify(boatswain_test_do, **opts)] + doers = wesDoers + [witDoer, bts, doing.doify(anchorer_test_do, **opts)] limit = 1.0 tock = 0.03125 @@ -60,7 +64,7 @@ def test_boatswain(seeder): assert bytes(delHby.db.getAes(dgkey)) == couple -def boatswain_test_do(tymth=None, tock=0.0, **opts): +def anchorer_test_do(tymth=None, tock=0.0, **opts): yield tock # enter context wesHab = opts["wesHab"] @@ -79,8 +83,8 @@ def boatswain_test_do(tymth=None, tock=0.0, **opts): witDoer.cues.popleft() msg = next(wesHab.db.clonePreIter(pre=palHab.pre)) - kvy = eventing.Kevery(db=delHby.db, local=False) - parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy) + kvy = eventing.Kevery(db=delHby.db, local=True) + parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy, local=True) while palHab.pre not in delHby.kevers: yield tock @@ -107,8 +111,8 @@ def boatswain_test_do(tymth=None, tock=0.0, **opts): couple = coring.Seqner(sn=palHab.kever.sn).qb64b + palHab.kever.serder.saidb msg = next(wesHab.db.clonePreIter(pre=palHab.pre, fn=1)) - kvy = eventing.Kevery(db=delHby.db, local=False) - parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy) + kvy = eventing.Kevery(db=delHby.db, local=True) + parsing.Parser().parseOne(ims=bytearray(msg), kvy=kvy, local=True) # Wait for the anchor. If we timeout before that happens, assertion in test will fail while delHby.db.getAes(dgkey) != couple: @@ -116,7 +120,7 @@ def boatswain_test_do(tymth=None, tock=0.0, **opts): def test_delegation_request(mockHelpingNowUTC): - with habbing.openHab(name="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): delpre = "EArzbTSWjccrTdNRsFUUfwaJ2dpYxu9_5jI2PJ-TRri0" serder = eventing.delcept(keys=["DUEFuPeaDH2TySI-wX7CY_uW5FF41LRu3a59jxg1_pMs"], delpre=delpre, @@ -124,15 +128,17 @@ def test_delegation_request(mockHelpingNowUTC): evt = hab.endorse(serder=serder) exn, atc = delegating.delegateRequestExn(hab=hab, delpre=delpre, evt=evt) - assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg' - b'-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' - b'-AABAACzeUyP6__0oDca-Oiv2iGXKghBw_8sI4ZHyyeMedvz0iZIIQYqJd2Zt7cDHRh7xBGWI85J_oOixLET3mFZUu0A') + assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAACzeUyP' + b'6__0oDca-Oiv2iGXKghBw_8sI4ZHyyeMedvz0iZIIQYqJd2Zt7cDHRh7xBGWI85J' + b'_oOixLET3mFZUu0A') assert exn.ked["r"] == '/delegate/request' assert exn.saidb == b'EHPkcmdLGql9_1WD0wl0OalYk8PcF4HMMd7gGi-iqfSe' - assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg' - b'-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' - b'-AABAACzeUyP6__0oDca-Oiv2iGXKghBw_8sI4ZHyyeMedvz0iZIIQYqJd2Zt7cDHRh7xBGWI85J_oOixLET3mFZUu0A') + assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAACzeUyP' + b'6__0oDca-Oiv2iGXKghBw_8sI4ZHyyeMedvz0iZIIQYqJd2Zt7cDHRh7xBGWI85J' + b'_oOixLET3mFZUu0A') data = exn.ked["a"] assert data["delpre"] == delpre embeds = exn.ked['e'] diff --git a/tests/app/test_directing.py b/tests/app/test_directing.py index 2a86227b5..2c39484e4 100644 --- a/tests/app/test_directing.py +++ b/tests/app/test_directing.py @@ -11,8 +11,11 @@ from hio.core.tcp import clienting, serving from keri import help # logger support -from keri.app import habbing, directing +from keri import core from keri.core import eventing, coring + +from keri.app import habbing, directing + from keri.demo import demoing @@ -25,7 +28,7 @@ def test_directing_basic(): raw = b"raw salt to test" # create bob signers and secrecies - bobSigners = coring.Salter(raw=raw).signers(count=8, path="bob", temp=True) + bobSigners = core.Salter(raw=raw).signers(count=8, path="bob", temp=True) bobSecrecies = [[signer.qb64] for signer in bobSigners] # bob inception transferable (nxt digest not empty) @@ -38,7 +41,7 @@ def test_directing_basic(): # create eve signers and secrecies - eveSigners = coring.Salter(raw=raw).signers(count=8, path="eve", temp=True) + eveSigners = core.Salter(raw=raw).signers(count=8, path="eve", temp=True) eveSecrecies = [[signer.qb64] for signer in eveSigners] # eve inception transferable (nxt digest not empty) @@ -172,7 +175,7 @@ def test_runcontroller_demo(): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path=name, temp=True)] diff --git a/tests/app/test_forwarding.py b/tests/app/test_forwarding.py index 7f94ae439..0fa1b3c5f 100644 --- a/tests/app/test_forwarding.py +++ b/tests/app/test_forwarding.py @@ -8,14 +8,17 @@ from hio.base import doing, tyming -from keri.app import forwarding, habbing, indirecting, storing +from keri import core from keri.core import coring, eventing, parsing, serdering + +from keri.app import forwarding, habbing, indirecting, storing + from keri.peer import exchanging def test_postman(seeder): with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab), \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64, temp=True) as wesHby, \ + habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64, temp=True) as wesHby, \ habbing.openHby(name="repTest", temp=True) as recpHby: mbx = storing.Mailboxer(name="wes", temp=True) @@ -29,15 +32,15 @@ def test_postman(seeder): recpIcp = recpHab.makeOwnEvent(sn=0) wesKvy = eventing.Kevery(db=wesHab.db, lax=False, local=False) - parsing.Parser().parse(ims=bytearray(recpIcp), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(recpIcp), kvy=wesKvy, local=True) assert recpHab.pre in wesKvy.kevers serder = serdering.SerderKERI(raw=recpIcp) rct = wesHab.receipt(serder) kvy = eventing.Kevery(db=hab.db) - parsing.Parser().parseOne(bytearray(recpIcp), kvy=kvy) - parsing.Parser().parseOne(bytearray(rct), kvy=kvy) + parsing.Parser().parseOne(bytearray(recpIcp), kvy=kvy, local=True) + parsing.Parser().parseOne(bytearray(rct), kvy=kvy, local=True) kvy.processEscrows() assert recpHab.pre in kvy.kevers @@ -65,7 +68,7 @@ def test_postman(seeder): doist.exit() msgs = [] - for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo", fn=0): + for _, topic, msg in mbx.cloneTopicIter(topic=recpHab.pre + "/echo"): msgs.append(msg) assert len(msgs) == 1 diff --git a/tests/app/test_grouping.py b/tests/app/test_grouping.py index 6141a0562..70b8f996a 100644 --- a/tests/app/test_grouping.py +++ b/tests/app/test_grouping.py @@ -5,7 +5,7 @@ """ from contextlib import contextmanager - +from keri import kering, core from keri.app import habbing, grouping, notifying from keri.core import coring, eventing, parsing, serdering from keri.vdr import eventing as veventing @@ -27,14 +27,14 @@ def test_counselor(): kev3 = eventing.Kevery(db=hab3.db, lax=True, local=False) icp1 = hab1.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3, local=True) icp2 = hab2.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3, local=True) icp3 = hab3.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2, local=True) smids = [hab1.pre, hab2.pre, hab3.pre] rmids = [hab1.pre, hab2.pre, hab3.pre] @@ -69,7 +69,7 @@ def test_counselor(): b'"a":[]}-AABBBBkMCMWP1Z2MMd6dBPlogRd1k6mv1joiHIyb8mXvp0H4kY0DHIPM' b'9O6udZ1Bbyf3klr4uGnLs07qcCcnKGI6GsH') - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 to process all sigs together counselor.processEscrows() @@ -84,7 +84,7 @@ def test_counselor(): migers = [hab1.kever.ndigers[0], hab2.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(smids=smids, rmids=rmids, isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -118,7 +118,7 @@ def test_counselor(): # Create group rotation from second participant - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() @@ -139,7 +139,7 @@ def test_counselor(): migers = [hab1.kever.ndigers[0], hab2.kever.ndigers[0], hab3.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(smids=smids, rmids=rmids, isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -175,7 +175,7 @@ def test_counselor(): # Create group rotation from second participant - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() @@ -195,7 +195,7 @@ def test_counselor(): migers = [hab1.kever.ndigers[0], hab3.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(smids=smids, rmids=rmids, isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith="2", nsith="2", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -229,7 +229,7 @@ def test_counselor(): # Create group rotation from second participant - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() @@ -250,13 +250,13 @@ def test_the_seven(): # All the Habs, this will come in handy later # Keverys so we can process each other's inception messages. - kev1 = eventing.Kevery(db=hab1.db, lax=True, local=False) - kev2 = eventing.Kevery(db=hab2.db, lax=True, local=False) - kev3 = eventing.Kevery(db=hab3.db, lax=True, local=False) - kev4 = eventing.Kevery(db=hab4.db, lax=True, local=False) - kev5 = eventing.Kevery(db=hab5.db, lax=True, local=False) - kev6 = eventing.Kevery(db=hab6.db, lax=True, local=False) - kev7 = eventing.Kevery(db=hab7.db, lax=True, local=False) + kev1 = eventing.Kevery(db=hab1.db) + kev2 = eventing.Kevery(db=hab2.db) + kev3 = eventing.Kevery(db=hab3.db) + kev4 = eventing.Kevery(db=hab4.db) + kev5 = eventing.Kevery(db=hab5.db) + kev6 = eventing.Kevery(db=hab6.db) + kev7 = eventing.Kevery(db=hab7.db) kevs = [kev1, kev2, kev3, kev4, kev5, kev6, kev7] icps = [hab1.makeOwnEvent(sn=0), @@ -271,7 +271,7 @@ def test_the_seven(): # Introduce everyone to each other by parsing each others ICP event into our keverys for (kev, icp) in [(kev, icp) for (kdx, kev) in enumerate(kevs) for (idx, icp) in enumerate(icps) if kdx != idx]: - parsing.Parser().parse(ims=bytearray(icp), kvy=kev) + parsing.Parser().parse(ims=bytearray(icp), kvy=kev, local=True) smids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre, hab5.pre, hab6.pre, hab7.pre] rmids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre, hab5.pre, hab6.pre, hab7.pre] @@ -313,7 +313,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBBAD108k4sWtYRv8jQaRbzX6kDebjdzFNVCh3N9cOAJqXV5IzmKdi60Cr0Eu' b'MaACskw0FCi73V2VX8BgFlxO8VIK') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab3 = hby3.makeGroupHab(group=f"{prefix}_group3", mhab=hab3, smids=smids, rmids=rmids, **inits) @@ -322,7 +322,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBCD6V2UkAovhY07MrJUNb-ICddDoyLde9i0FWclxfs7jes01YUEihfgbGERF' b'dKDR4kSr4WF3AskrZOPvMuXipAgP') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab4 = hby4.makeGroupHab(group=f"{prefix}_group4", mhab=hab4, smids=smids, rmids=rmids, **inits) @@ -331,7 +331,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBDBCZuZSFWy0tFshGny1pTR47GphDljd0SShmGRpUSpBX_BeHB1tdIObizaA' b'4GMoOcZ2sOWIe6muJPF_RaoKedYE') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab5 = hby5.makeGroupHab(group=f"{prefix}_group5", mhab=hab5, smids=smids, rmids=rmids, **inits) @@ -340,7 +340,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBEBsR6_hPId3H8fFG8EfevQVji8MsLAC72MjkkRxJp3h9v1vyFS1hAGGGxno' b'F5xSHOnpBpPwjMJwOCurAa3VrNAD') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab6 = hby6.makeGroupHab(group=f"{prefix}_group6", mhab=hab6, smids=smids, rmids=rmids, **inits) @@ -349,7 +349,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBFCi5hK6Ax4aBNsdoUkh7Q_CcSWJfpwkeF68aCO34J3BDN7k483lOxiyj6pl' b'8TQIQ7VJLBkoRscUMi_mls9jbpcD') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception ghab7 = hby7.makeGroupHab(group=f"{prefix}_group7", mhab=hab7, smids=smids, rmids=rmids, **inits) @@ -358,7 +358,7 @@ def test_the_seven(): assert evt[serd.size:] == (b'-AABBGCtPvRj00vEfT5Po6eH50DWfBWwAcQgvBaJ7LlYT7kQswkl_r-K9Lsxi5tm' b'Pvsb2xFtcMJkFf-BxamGhFo9OOcD') assert serd.raw == raw - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 to process all sigs together @@ -376,8 +376,8 @@ def test_the_seven(): hab5.kever.ndigers[0], hab6.kever.ndigers[0], hab7.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(smids=smids, rmids=rmids, isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', - toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', + toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -409,7 +409,7 @@ def test_the_seven(): msg = eventing.messagize(serder=serder, sigers=sigers) assert msg[serder.size:] == (b'-AABABAzvHN7yC3581dp9DxFXrKuXGP_62r_pzNMXL20T6RaPQASXvnBn6sKJ78z' b'KM9o499Zaz76j940nBoMT-yb9i8N') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception # Now sign the group ROT with Hab3 and parse into Kev1. This should commit the event sigers = hab3.mgr.sign(serder.raw, verfers=hab3.kever.verfers, indexed=True, indices=[2]) @@ -417,7 +417,7 @@ def test_the_seven(): assert msg[serder.size:] == (b'-AABACB6z6LrzBAgpnrCopgiGxuki3sE-KAfY8t_rFq-2dIcQxRF4iCqCYNPKM9D' b'NbZbA1WDaQ72enSsR2UWMftX2kYD') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() # Get the rest of the way through counselor. @@ -440,8 +440,8 @@ def test_the_seven(): hab5.kever.ndigers[0], hab6.kever.ndigers[0], hab7.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab.rotate(smids=smids, rmids=rmids, isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', - toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab.rotate(isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3", "1/3", "1/3", "1/3", "1/3"]', + toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor.start(ghab=ghab, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -474,7 +474,7 @@ def test_the_seven(): msg = eventing.messagize(serder=serder, sigers=sigers) assert msg[serder.size:] == (b'-AABABC4sYnDXCpO87BMXO21ofqHZKntPSdEXlBPlq1H8NOHD3KV-GHGWrXyrElK' b'BkQNBbNr9_yg-nSnBq7N9rAxEFcK') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception # Now sign the group ROT with Hab3 and parse into Kev1. This should commit the event sigers = hab3.mgr.sign(serder.raw, verfers=hab3.kever.verfers, indexed=True, indices=[2]) @@ -482,7 +482,7 @@ def test_the_seven(): assert msg[serder.size:] == (b'-AABACAXyUueUfXC-ccUxBZTgnyHTXOy1wUYgQrhlk8FMJGQPiaOOdAzhaW71JeF' b'0By8Se-tKKuPP1xG41DblgXIwNkE') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev1) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev1, local=True) # parse second signed group inception kev1.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor.processEscrows() # Get the rest of the way through counselor. @@ -502,7 +502,7 @@ def test_the_seven(): msgs = [hab1.replay(), hab2.replay(), hab3.replay(), ghab.replay()] kevs = [kev4, kev5, kev6, kev7] for (kev, msg) in [(kev, msg) for kev in kevs for msg in msgs]: - parsing.Parser().parse(ims=bytearray(msg), kvy=kev) + parsing.Parser().parse(ims=bytearray(msg), kvy=kev, local=True) assert kev4.kevers[ghab.pre] is not None assert kev5.kevers[ghab.pre] is not None @@ -519,8 +519,8 @@ def test_the_seven(): migers = [hab4.kever.ndigers[0], hab5.kever.ndigers[0], hab6.kever.ndigers[0]] prefixer = coring.Prefixer(qb64=ghab.pre) seqner = coring.Seqner(sn=ghab.kever.sn + 1) - rot = ghab4.rotate(smids=smids, rmids=rmids, isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3"]', - toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rot = ghab4.rotate(isith='["1/3", "1/3", "1/3"]', nsith='["1/3", "1/3", "1/3"]', + toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers, smids=smids, rmids=rmids) rserder = serdering.SerderKERI(raw=rot) counselor4.start(ghab=ghab4, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) @@ -549,7 +549,7 @@ def test_the_seven(): msg = eventing.messagize(serder=serder, sigers=sigers) assert msg[serder.size:] == (b'-AAB2AABAEDSs99oM-KOhJ8q3H8lqGqPE3EvZxCHvCjZFvWHLzhqm91YlcskGqvK' b'8DwCg9dj8wRZP54ienzD52EIKvJWWh4J') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev4) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev4, local=True) # parse second signed group inception # Now sign the group ROT with Hab6 and parse into Kev4. This should commit the event sigers = hab6.mgr.sign(serder.raw, verfers=hab6.kever.verfers, indexed=True, indices=[2], ondices=[5]) @@ -557,7 +557,7 @@ def test_the_seven(): assert msg[serder.size:] == (b'-AAB2AACAFBNVTM0Gw4rSd-S5HQ_KpmBfDedi7XNvB24ijMjQaekIfKlcdguPS8p' b'ax9ht7EE3SiTj9fSO_3f4SVUfJMPmHIK') - parsing.Parser().parse(ims=bytearray(msg), kvy=kev4) # parse second signed group inception + parsing.Parser().parse(ims=bytearray(msg), kvy=kev4, local=True) # parse second signed group inception kev4.processEscrows() # Run escrows for Kevery1 so he processes all sigs together counselor4.processEscrows() # Get the rest of the way through counselor. @@ -582,14 +582,14 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): kev3 = eventing.Kevery(db=hab3.db, lax=True, local=False) icp1 = hab1.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2) - parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(icp1), kvy=kev3, local=True) icp2 = hab2.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp2), kvy=kev3, local=True) icp3 = hab3.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1) - parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev1, local=True) + parsing.Parser().parse(ims=bytearray(icp3), kvy=kev2, local=True) smids = [hab1.pre, hab2.pre, hab3.pre] rmids = None @@ -616,13 +616,13 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): sigs.extend(bytes(hab3.db.getSigs(dgkey)[0])) evt = bytearray(eraw) - evt.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=3).qb64b) # attach cnt + evt.extend(core.Counter(core.Codens.ControllerIdxSigs, + count=3, gvrsn=kering.Vrsn_1_0).qb64b) # attach cnt evt.extend(sigs) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev3) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev2) - parsing.Parser().parse(ims=bytearray(evt), kvy=kev1) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev3, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev2, local=True) + parsing.Parser().parse(ims=bytearray(evt), kvy=kev1, local=True) assert ghab1.pre in kev1.kevers assert ghab1.pre in kev2.kevers @@ -632,17 +632,18 @@ def openMultiSig(prefix="test", salt=b'0123456789abcdef', temp=True, **kwa): def test_multisig_incept(mockHelpingNowUTC): - with habbing.openHab(name="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): aids = [hab.pre, "EfrzbTSWjccrTdNRsFUUfwaJ2dpYxu9_5jI2PJ-TRri0"] exn, atc = grouping.multisigInceptExn(hab=hab, smids=aids, rmids=aids, icp=hab.makeOwnEvent(sn=hab.kever.sn)) assert exn.ked["r"] == '/multisig/icp' assert exn.saidb == b'EJ6Kl50IBicAa8zND_3wMSQ5itw555V7NKid9y1SKobe' - assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg' - b'-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' - b'-AABAACL4cf7LxzKJgaJbb7wWHLuTfj3wManDV0SW7euFNZDiEhD1kUiP3_wtOIfqB_ZsEceE4oIgOOZwFROyrcf9ScB' - b'-LAa5AACAA-e-icp-AABAACihaKoLnoXxRoxGbFfOy67YSh6UxtgjT2oxupnLDz2FlhevGJKTMObbdex9f0Hqob6uTavSJvsXf5RzitskkkC') + assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAACL4cf7' + b'LxzKJgaJbb7wWHLuTfj3wManDV0SW7euFNZDiEhD1kUiP3_wtOIfqB_ZsEceE4oI' + b'gOOZwFROyrcf9ScB-LAa5AACAA-e-icp-AABAACihaKoLnoXxRoxGbFfOy67YSh6' + b'UxtgjT2oxupnLDz2FlhevGJKTMObbdex9f0Hqob6uTavSJvsXf5RzitskkkC') data = exn.ked["a"] assert data["smids"] == aids assert "icp" in exn.ked['e'] @@ -661,10 +662,11 @@ def test_multisig_rotate(mockHelpingNowUTC): exn, atc = grouping.multisigRotateExn(ghab=ghab1, smids=ghab1.smids, rmids=ghab1.rmids, rot=rot) assert exn.ked["r"] == '/multisig/rot' - assert exn.saidb == b'EC2IKkvJh6_Ukx-ZWP20qyHPWpXYfZdCQkydA9HwYE9c' - assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAAAAAAAAAAEH__mobl7NDyyQCB1DoLK' - b'-OPSueraPtZAlWEjfOYkaba-AABAABxikcUcQLQyCuOfQXYBeyFd3hzMaaZ_wHV_KPPX8DyFcold4P8mdGC' - b'-meFY9P7qoJd3lPA1khblmqY5jhK2kAL') + assert exn.saidb == b'EL4LeEHvTiOxs1UDNTv5qWxCYVYojdpEMfKI62O-UsPm' + assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAA' + b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAACH_qI1' + b'JebS_iehZT6XmvxylpOy2hS2BjO41e4mNmscSBdun2MyGk82SC-rHfQfvDJZlRRw' + b'NhLw-pKKKxql8wUF') data = exn.ked["a"] assert data["smids"] == ghab1.smids @@ -673,14 +675,14 @@ def test_multisig_rotate(mockHelpingNowUTC): def test_multisig_rotate_new_group_member_updates_smids(mockHelpingNowUTC): # Create a multisig with three members, test_1, test_2, and test_3 - with openMultiSig(prefix="test") as ((hby1, ghab1), (hby2, ghab2), (hby3, ghab3)): + with openMultiSig(prefix="smidstest") as ((hby1, ghab1), (hby2, ghab2), (hby3, ghab3)): # Create a new member, test_4 - with habbing.openHab(name="test_4", salt=b'0123456789abcdef', transferable=True, temp=True) as (hby4, hab4): + with habbing.openHab(name="smidstest_4", salt=b'0123456789abcdef', transferable=True, temp=True) as (hby4, hab4): icp4 = hab4.makeOwnEvent(sn=0) # Get test_4's inception event to introduce to group members - hab1 = hby1.habByName("test_1") - hab2 = hby2.habByName("test_2") - hab3 = hby3.habByName("test_3") + hab1 = hby1.habByName("smidstest_1") + hab2 = hby2.habByName("smidstest_2") + hab3 = hby3.habByName("smidstest_3") # Create member Kevery instances to parse each other's events and update their keystate kev1 = eventing.Kevery(db=hab1.db, lax=True, local=False) kev2 = eventing.Kevery(db=hab2.db, lax=True, local=False) @@ -722,7 +724,7 @@ def test_multisig_rotate_new_group_member_updates_smids(mockHelpingNowUTC): rmids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre] # make group hab for test_4 - ghab4 = hby4.joinGroupHab(hab4.pre, group="test_group4", mhab=hab4, smids=smids, rmids=rmids) + ghab4 = hby4.joinGroupHab(hab4.pre, group="smidstest_group4", mhab=hab4, smids=smids, rmids=rmids) isith = '["1/4", "1/4", "1/4", "1/4"]' nsith = '["1/4", "1/4", "1/4", "1/4"]' @@ -761,7 +763,7 @@ def test_multisig_rotate_new_group_member_updates_smids(mockHelpingNowUTC): assert ghab4.smids == smids assert ghab4.rmids == rmids hby1.loadHabs() - ghab1 = hby1.habByName("test_group1") # reload hab to get updated smids and rmids values + ghab1 = hby1.habByName("smidstest_group1") # reload hab to get updated smids and rmids values assert ghab1.smids == smids assert ghab1.rmids == rmids @@ -774,11 +776,11 @@ def test_multisig_interact(mockHelpingNowUTC): assert exn.ked["r"] == '/multisig/ixn' assert exn.saidb == b'EDF8o6SK-s2jxUVnlGtqAVtXTF-wyZ26c0dUsS5p766q' - assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAAAAAAAAAAEH__mobl7NDyyQCB1DoLK' - b'-OPSueraPtZAlWEjfOYkaba' - b'-AABAABFfU5so86inNogCPN7Ko8WXvkMKeiUKPScQ3FYrVmngNpVmW8xmhOTfixuWFlLcQPjEf3bRQhvNvx7azcI_vwB' - b'-LAa5AACAA-e-ixn-AABAABG58m7gibjdrQ8YU' - b'-8WQ8A70nctYekYr3xdfZ5WgDQOD0bb9pI7SuuaJvzfAQisLAYQnztA82pAo1Skhf1vQwD') + assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAA' + b'AAAAAAAAEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba-AABAABFfU5s' + b'o86inNogCPN7Ko8WXvkMKeiUKPScQ3FYrVmngNpVmW8xmhOTfixuWFlLcQPjEf3b' + b'RQhvNvx7azcI_vwB-LAa5AACAA-e-ixn-AABAABG58m7gibjdrQ8YU-8WQ8A70nc' + b'tYekYr3xdfZ5WgDQOD0bb9pI7SuuaJvzfAQisLAYQnztA82pAo1Skhf1vQwD') data = exn.ked["a"] assert data["smids"] == ghab1.smids assert data["gid"] == ghab1.pre @@ -793,21 +795,21 @@ def test_multisig_registry_incept(mockHelpingNowUTC, mockCoringRandomNonce): usage="Issue vLEI Credentials") assert exn.ked["r"] == '/multisig/vcp' - assert exn.saidb == b'ELTlVFjqhqLkGBqItC9P6RADjranADW8FwD7nnz5ngwO' - assert atc == (b'-FABEH__mobl7NDyyQCB1DoLK-OPSueraPtZAlWEjfOYkaba0AAAAAAAAAAAAAAAAAAAAAAAEH__mobl7NDyyQCB1DoLK' - b'-OPSueraPtZAlWEjfOYkaba' - b'-AABAABY2UZSi_FQRViWfk_wdBmbgPUus1PtJzBPUDpfKEYvhHhsT6IB7z3IswPlrwUc0rTwjN1ON9ssFoTtTlMJqG8K' - b'-LAa5AACAA-e-anc-AABAAD2mK9ICW9x1' - b'-0NZGkEDOcAbZ58VWK9LOTwyN2lSfHr2zY638P1SBStoh8mjgy7nOTGMyujOXMKvF_ZDeQ_ISYA') + assert exn.saidb == b'EBum6f9SwkUUjQTl_vDplKs7L-shzQT6fS5jJlzdP9PP' + assert atc == (b'-FABEDEf72ZZ9mhpT1Xz-_YkXl7cg93sjZUFLIsxaFNTbXQO0AAAAAAAAAAAAAAA' + b'AAAAAAAAEDEf72ZZ9mhpT1Xz-_YkXl7cg93sjZUFLIsxaFNTbXQO-AABAAAS5k5D' + b'9jH0rbS6jCtZIPyTJRS2l8TZBnChwG8try3kZUJuiAPoBLo7UuhFYmZlpTZ6MfSg' + b'cDS7XNg0ETj6L3QF-LAa5AACAA-e-anc-AABAABXlwkzbp_tC4MEbx1Uyny1o7dB' + b'GHrYjU3u90Mhv2GtrIGG-7va1jZnlXef2R_LM4TRN8_XjmpLv1skcJaM90UB') data = exn.ked["a"] - assert data == {'gid': 'EERn_laF0qwP8zTBGL86LbF84J0Yh2IvQSRskH3BZZiy', + assert data == {'gid': 'EEVG5a8c88Fg9vH-6zQP6gJdc4LxVbUTRydx-JhpDcob', 'usage': 'Issue vLEI Credentials'} assert "vcp" in exn.ked["e"] assert "anc" in exn.ked["e"] def test_multisig_incept_handler(mockHelpingNowUTC): - with habbing.openHab(name="test0", temp=True) as (hby, hab): + with habbing.openHab(name="test0", temp=True, salt=b'0123456789abcdef') as (hby, hab): aids = [hab.pre, "EfrzbTSWjccrTdNRsFUUfwaJ2dpYxu9_5jI2PJ-TRri0"] exn, atc = grouping.multisigInceptExn(hab=hab, smids=aids, rmids=aids, icp=hab.makeOwnEvent(sn=hab.kever.sn)) @@ -898,3 +900,72 @@ def test_multisig_interact_handler(mockHelpingNowUTC): prefixers = hby1.db.maids.get(keys=(esaid,)) assert len(prefixers) == 1 assert prefixers[0].qb64 == ghab2.mhab.pre + +def test_multisig_join(mockHelpingNowUTC): + with openMultiSig(prefix="test") as ((hby1, ghab1), (hby2, ghab2), (hby3, ghab3)): + with habbing.openHab(name="test_4", salt=b'0123456789abcdef', transferable=True, temp=True) as (hby4, hab4): + hab1, hab2, hab3 = ghab1.mhab, ghab2.mhab, ghab3.mhab + + # Share fourth member's inception with existing group + icp4 = hab4.makeOwnEvent(sn=0) + parsing.Parser().parse(ims=bytearray(icp4), kvy=hby1.kvy, local=True) + parsing.Parser().parse(ims=bytearray(icp4), kvy=hby2.kvy, local=True) + parsing.Parser().parse(ims=bytearray(icp4), kvy=hby3.kvy, local=True) + + # Rotate the group to add the fourth member (3 current, 4 next) + counselor = grouping.Counselor(hby=hby1) + hab1.rotate() + hab2.rotate() + hab3.rotate() + merfers = [hab1.kever.verfers[0], hab2.kever.verfers[0], hab3.kever.verfers[0]] + migers = [hab1.kever.ndigers[0], hab2.kever.ndigers[0], hab3.kever.ndigers[0], hab4.kever.ndigers[0]] + prefixer = coring.Prefixer(qb64=ghab1.pre) + seqner = coring.Seqner(sn=ghab1.kever.sn + 1) + rot = ghab1.rotate(isith="3", nsith="4", toad=0, cuts=list(), adds=list(), verfers=merfers, digers=migers) + rserder = serdering.SerderKERI(raw=rot) + counselor.start(ghab=ghab1, prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) + + val = hby1.db.gpse.get(keys=(ghab1.pre,)) + (seqner, saider) = val[0] + + key = dbing.dgKey(ghab1.pre, saider.qb64b) # digest key + evt = hby1.db.getEvt(key=key) + serder = serdering.SerderKERI(raw=bytes(evt)) + assert serder is not None + sigers = hab2.mgr.sign(serder.raw, verfers=hab2.kever.verfers, indexed=True, indices=[1]) + msg = eventing.messagize(serder=serder, sigers=sigers) + parsing.Parser().parse(ims=bytearray(msg), kvy=hby1.kvy, local=True) + + sigers = hab3.mgr.sign(serder.raw, verfers=hab3.kever.verfers, indexed=True, indices=[2]) + msg = eventing.messagize(serder=serder, sigers=sigers) + parsing.Parser().parse(ims=bytearray(msg), kvy=hby1.kvy, local=True) + hby1.kvy.processEscrows() + counselor.processEscrows() + assert counselor.complete(prefixer=prefixer, seqner=seqner, saider=coring.Saider(qb64=rserder.said)) + + # Load member and group KELs into hby4 + for msgs in [hab1.replay(), hab2.replay(), hab3.replay()]: + parsing.Parser().parse(ims=bytearray(msgs), kvy=hby4.kvy, local=True) + parsing.Parser().parse(ims=bytearray(ghab1.replay()), kvy=hby4.kvy, local=True) + + smids = [hab1.pre, hab2.pre, hab3.pre, hab4.pre] + hby4.joinGroupHab(ghab1.pre, group="test_group_4", mhab=hab4, smids=smids, rmids=None) + + ghab4 = hby4.habByName("test_group_4") + + assert ghab4.pre == ghab1.pre + assert ghab4.mhab.pre == hab4.pre + assert ghab4.smids == smids + assert ghab4.kever.sn == 1 + assert ghab4.inited is True + assert len(ghab4.kever.verfers) == 3 + assert len(ghab4.kever.ndigers) == 4 + assert ghab4.pre in hby4.habs + + # Verify that the group is consistent after a reload + hby4.loadHabs() + ghab4 = hby4.habByName("test_group_4") + assert ghab4.pre == ghab1.pre + assert ghab4.mhab.pre == hab4.pre + assert ghab4.name == "test_group_4" + diff --git a/tests/app/test_habbing.py b/tests/app/test_habbing.py index 69e0d65a4..4fb896f87 100644 --- a/tests/app/test_habbing.py +++ b/tests/app/test_habbing.py @@ -11,14 +11,17 @@ from hio.base import doing from keri import kering + from keri import help +from keri.help import helping + +from keri import core +from keri.core import coring, eventing, parsing + from keri.app import habbing, keeping, configing -from keri.core.coring import MtrDex from keri.db import basing -from keri.core import coring, eventing, parsing -from keri.help import helping -from keri.peer import exchanging -from keri.vdr.eventing import incept + + def test_habery(): @@ -26,8 +29,8 @@ def test_habery(): Test Habery class """ # test default - default_salt = coring.Salter(raw=b'0123456789abcdef').qb64 - hby = habbing.Habery(temp=True) + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + hby = habbing.Habery(temp=True, salt=default_salt) assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -58,7 +61,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low hby.cf.close(clear=True) hby.db.close(clear=True) @@ -73,13 +76,13 @@ def test_habery(): assert len(bran) == 21 # these are the ones generated by Manager for the given bran # bran = coring.MtrDex.Salt_128 + 'A' + bran[:21] # qb64 salt for seed - # signer = coring.Salter(qb64=bran).signer(transferable=False, tier=tier, temp=temp) + # signer = core.Salter(qb64=bran).signer(transferable=False, tier=tier, temp=temp) # seed = signer.qb64 # aeid = signer.verfer.qb64 # lest it remove encryption seed4bran = 'ALQVu9AjGW3JrIzX0UHm2awGCbDXcsLzy-vAE649Fz1j' aeid4seed = 'BHRYV_5a1AlibCrXFG_KDD9rC6aXx9cb0sR968NL80VI' - hby = habbing.Habery(bran=bran, temp=True) + hby = habbing.Habery(bran=bran, temp=True, salt=default_salt) assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -91,7 +94,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -129,7 +132,7 @@ def test_habery(): # setup habery hby = habbing.Habery(name=name, base=base, ks=ks, db=db, cf=cf, temp=temp, - bran=bran) + bran=bran, salt=default_salt) hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer assert hby.name == "main" @@ -158,7 +161,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -183,7 +186,7 @@ def test_habery(): temp = True # setup habery with resources - hby = habbing.Habery(name=name, base=base, temp=temp, bran=bran, free=True) + hby = habbing.Habery(name=name, base=base, temp=temp, bran=bran, free=True, salt=default_salt) hbyDoer = habbing.HaberyDoer(habery=hby) # setup doer conf = hby.cf.get() @@ -219,7 +222,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -237,7 +240,7 @@ def test_habery(): assert not os.path.exists(hby.db.path) assert not os.path.exists(hby.ks.path) - with habbing.openHby() as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -267,7 +270,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -284,7 +287,7 @@ def test_habery(): assert not os.path.exists(hby.ks.path) bran = "MyPasscodeARealSecret" - with habbing.openHby(bran=bran) as hby: + with habbing.openHby(bran=bran, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: assert hby.name == "test" assert hby.base == "" assert hby.temp @@ -316,7 +319,7 @@ def test_habery(): assert hby.mgr.salt == default_salt assert hby.mgr.pidx == 1 assert hby.mgr.algo == keeping.Algos.salty - assert hby.mgr.tier == coring.Tiers.low + assert hby.mgr.tier == core.Tiers.low assert hby.rtr.routes assert hby.rvy.rtr == hby.rtr @@ -345,7 +348,7 @@ def test_make_load_hab_with_habery(): name = "sue" suePre = 'ELF1S0jZkyQx8YtHaPLu-qyFmrkcykAiEW8twS-KPSO1' # with temp=True - with habbing.openHby() as hby: # default is temp=True on openHab + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True on openHab hab = hby.makeHab(name=name) assert isinstance(hab, habbing.Hab) assert hab.pre in hby.habs @@ -389,7 +392,7 @@ def test_make_load_hab_with_habery(): suePre = 'EAxe215BJ4Iy9r0mfoMEGVmHW8A4Avk3RYBC1A1_DZam' # with temp=False bobPre = 'ENya5E5pvc6MVCe75huDK0QQhE4_64J55vCn4aKdXhR9' # with temp=False - with habbing.openHby(base=base, temp=False) as hby: # default is temp=True + with habbing.openHby(base=base, temp=False, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True assert hby.cf.path.endswith("keri/cf/hold/test.json") assert hby.db.path.endswith("keri/db/hold/test") assert hby.ks.path.endswith('keri/ks/hold/test') @@ -563,7 +566,7 @@ def test_habery_reinitialization(): def test_habery_signatory(): - with habbing.openHby() as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: signer = hby.signator assert signer is not None @@ -602,11 +605,11 @@ def test_habery_reconfigure(mockHelpingNowUTC): # use same salter but with different path from name for each # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - # salter = coring.Salter(raw=raw) + # salter = core.Salter(raw=raw) # salt = salter.qb64 # assert salt == '0ABaqPLVOa6fpVnAKcmwhIdQ' - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 cname = "tam" # controller name cbase = "main" # controller base shared @@ -717,7 +720,7 @@ def test_habery_reconfigure(mockHelpingNowUTC): def test_namespaced_habs(): - with habbing.openHby() as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" @@ -730,8 +733,7 @@ def test_namespaced_habs(): nshab = hby.makeHab(name="test2", ns="agent") assert nshab.pre == "EErXOolQNmKrTMKfXdQ1sj8YsgZZe4wMXZwsX-j1V6Dd" - assert len(hby.habs) == 1 - assert len(hby.namespaces) == 1 + assert len(hby.habs) == 2 assert len(hby.prefixes) == 2 found = hby.habByName(name="test2") @@ -745,11 +747,8 @@ def test_namespaced_habs(): nshab = hby.makeHab(name="test.3", ns="agent") assert nshab.pre == "EG5FUOzW_KKVB8JGlNGoZAADDC8cZ6Jt079nLEaFnYcg" - assert len(hby.habs) == 1 - assert len(hby.namespaces) == 1 + assert len(hby.habs) == 3 assert len(hby.prefixes) == 3 - ns = hby.namespaces['agent'] - assert len(ns) == 2 # '.' characters not allowed in namespace names with pytest.raises(kering.ConfigurationError): @@ -776,14 +775,9 @@ def test_namespaced_habs(): assert pre in hby.db.kevers # read through cache assert pre in hby.db.prefixes - assert len(hby.habs) == 2 + assert len(hby.habs) == 5 assert len(hby.db.prefixes) == 5 - agent = hby.namespaces["agent"] - assert len(agent) == 2 - ctrl = hby.namespaces["controller"] - assert len(ctrl) == 1 - found = hby.habByName(name=name) assert found.pre == opre found = hby.habByName(name="test.1") @@ -800,7 +794,7 @@ def test_namespaced_habs(): def test_make_other_event(): - with habbing.openHby() as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" @@ -843,10 +837,10 @@ def test_hab_by_pre(): # Only habs in default namespace are in hby.habs assert hab1.pre in hby.habs assert hab2.pre in hby.habs - assert hab3.pre not in hby.habs - assert hab4.pre not in hby.habs - assert hab5.pre not in hby.habs - assert hab6.pre not in hby.habs + assert hab3.pre in hby.habs + assert hab4.pre in hby.habs + assert hab5.pre in hby.habs + assert hab6.pre in hby.habs assert hby.habByPre("EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3") is None @@ -857,22 +851,11 @@ def test_hab_by_pre(): assert hby.habByPre(pre=hab5.pre) == hab5 assert hby.habByPre(pre=hab6.pre) == hab6 - assert "one" in hby.namespaces - assert hab3.pre in hby.namespaces["one"] - assert hab4.pre in hby.namespaces["one"] - assert hab1.pre not in hby.namespaces["one"] - assert hab2.pre not in hby.namespaces["one"] - assert "two" in hby.namespaces - assert hab5.pre in hby.namespaces["two"] - assert hab6.pre in hby.namespaces["two"] - assert hab1.pre not in hby.namespaces["two"] - assert hab2.pre not in hby.namespaces["two"] - def test_postman_endsfor(): - with habbing.openHby(name="test", temp=True) as hby, \ - habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64, temp=True) as wesHby, \ - habbing.openHab(name="agent", temp=True) as (agentHby, agentHab): + with habbing.openHby(name="test", temp=True, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby, \ + habbing.openHby(name="wes", temp=True, salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHab(name="agent", temp=True, salt=b'0123456789abcdef') as (agentHby, agentHab): wesHab = wesHby.makeHab(name='wes', isith="1", icount=1, transferable=False) assert not wesHab.kever.prefixer.transferable @@ -892,7 +875,7 @@ def test_postman_endsfor(): kvy = eventing.Kevery(db=hab.db, lax=False, local=False) icpMsg = hab.makeOwnInception() rctMsgs = [] # list of receipts from each witness - parsing.Parser().parse(ims=bytearray(icpMsg), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(icpMsg), kvy=wesKvy, local=True) assert wesKvy.kevers[hab.pre].sn == 0 # accepted event assert len(wesKvy.cues) >= 1 # assunmes includes queued receipt cue # better to find cue in cues and confirm exactly @@ -901,11 +884,11 @@ def test_postman_endsfor(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=kvy, local=True) assert wesHab.pre in kvy.kevers agentIcpMsg = agentHab.makeOwnInception() - parsing.Parser().parse(ims=bytearray(agentIcpMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(agentIcpMsg), kvy=kvy, local=True) assert agentHab.pre in kvy.kevers msgs = bytearray() diff --git a/tests/app/test_httping.py b/tests/app/test_httping.py index 7987252cb..231a1c065 100644 --- a/tests/app/test_httping.py +++ b/tests/app/test_httping.py @@ -56,17 +56,22 @@ def test_parse_cesr_request(): assert cr.attachments == "-H000000000" +class MockRequester: + path = '/' + + class MockClient: def __init__(self): self.args = [] + self.requester = MockRequester() def request(self, **kwargs): self.args.append(kwargs) def test_create_cesr_request(mockHelpingNowUTC): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): wit = "BGKVzj4ve0VSd8z_AmvhLg4lqcC_9WYX90k03q-R_Ydo" regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=hab.pre, name="test") @@ -114,7 +119,7 @@ def test_create_cesr_request(mockHelpingNowUTC): def test_stream_cesr_request(mockHelpingNowUTC): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): wit = "BGKVzj4ve0VSd8z_AmvhLg4lqcC_9WYX90k03q-R_Ydo" regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=hab.pre, name="test") diff --git a/tests/app/test_indirecting.py b/tests/app/test_indirecting.py index 7f85e65c2..6118cd051 100644 --- a/tests/app/test_indirecting.py +++ b/tests/app/test_indirecting.py @@ -6,18 +6,20 @@ import json import time -import pytest import falcon from falcon import testing - import hio +import pytest + from hio.core import http from hio.base import doing, tyming from hio.help import decking from keri import kering +from keri import core from keri.app import indirecting, storing, habbing, agenting -from keri.core import serdering, coring +from keri.metric import EscrowEnd +from keri.vdr import viring def test_mailbox_iter(): @@ -106,12 +108,12 @@ def test_mailbox_multiple_iter(): def test_qrymailbox_iter(): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): assert hab.pre == 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' icp = hab.makeOwnInception() - icpSrdr = serdering.SerderKERI(raw=icp) + icpSrdr = core.serdering.SerderKERI(raw=icp) qry = hab.query(pre=hab.pre, src=hab.pre, route="/mbx") - srdr = serdering.SerderKERI(raw=qry) + srdr = core.serdering.SerderKERI(raw=qry) cues = decking.Deck() mbx = storing.Mailboxer(temp=True) @@ -158,17 +160,20 @@ def test_qrymailbox_iter(): def test_wit_query_ends(seeder): - with habbing.openHby(name="wes", salt=coring.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ - habbing.openHby(name="pal", salt=coring.Salter(raw=b'0123456789abcdef').qb64) as palHby: + with habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, tcpPort=5634, httpPort=5644) + # Pull the reger out of the Doers so the reger is reused and does not trigger an LMDB error on reuse + wesReger = next(doer.baser for doer in wesDoers + if isinstance(getattr(doer, "baser", None), viring.Reger)) witDoer = agenting.Receiptor(hby=palHby) wesHab = wesHby.habByName(name="wes") seeder.seedWitEnds(palHby.db, witHabs=[wesHab], protocols=[kering.Schemes.http]) app = falcon.App() - query_endpoint = indirecting.QueryEnd(wesHab) + query_endpoint = indirecting.QueryEnd(wesHab, reger=wesReger) app.add_route("/query", query_endpoint) wesClient = testing.TestClient(app) @@ -263,7 +268,79 @@ def wit_querier_test_do(tymth=None, tock=0.0, **opts): assert res.headers['Content-Type'] == "application/json" assert "unkown query type" in res.text +def test_wit_allowlist(seeder): + with habbing.openHby(name="wes", salt=core.Salter(raw=b'wess-the-witness').qb64) as wesHby, \ + habbing.openHby(name="pal", salt=core.Salter(raw=b'0123456789abcdef').qb64) as palHby: + + wesHab = wesHby.makeHab(name="wes", transferable=False) + + # Allowed (allowlisted) participant hab + palAllowed = palHby.makeHab(name="palAllowed", wits=[wesHab.pre], transferable=True) + # Denied (not allowlisted) participant hab + palDenied = palHby.makeHab(name="palDenied", wits=[wesHab.pre], transferable=True) + + aids = [palAllowed.pre] + + wesDoers = indirecting.setupWitness(alias="wes", hby=wesHby, aids=aids, tcpPort=5634, httpPort=5644) + witDoer = agenting.Receiptor(hby=palHby) + + seeder.seedWitEnds(palHby.db, witHabs=[wesHab], protocols=[kering.Schemes.http]) + + opts = dict( + wesHab=wesHab, + palHby=palHby, + witDoer=witDoer, + palAllowed=palAllowed, + palDenied=palDenied, + aids=aids, + ) + testDo = AllowlistTestDoer(**opts) + doers = wesDoers + [witDoer, testDo] + + limit = 1.0 + tock = 0.03125 + doist = doing.Doist(tock=tock, limit=limit, doers=doers) + doist.enter() + + while not testDo.done: + doist.recur() + time.sleep(doist.tock) + + assert doist.limit == limit + doist.exit() + +class AllowlistTestDoer(doing.Doer): + def __init__(self, **opts): + self.options = opts + super(AllowlistTestDoer, self).__init__(**opts) + + def recur(self, tyme=0.0, deeds=None, **kwa): + wesHab = self.options["wesHab"] + witDoer = self.options["witDoer"] + palAllowed = self.options["palAllowed"] + palDenied = self.options["palDenied"] + + # Trigger witness receipting for both allowed and denied AIDs + if not any(msg.get('pre') == palAllowed.pre for msg in witDoer.msgs): + witDoer.msgs.append(dict(pre=palAllowed.pre)) + if not any(msg.get('pre') == palDenied.pre for msg in witDoer.msgs): + witDoer.msgs.append(dict(pre=palDenied.pre)) + + # Wait until both have been processed (one cue each) + while len(witDoer.cues) < 2: + yield self.tock + + # Allowed pre should have at least one event stored on witness + allowed_iter = wesHab.db.clonePreIter(pre=palAllowed.pre) + allowed_msg = next(allowed_iter) # should NOT raise + assert isinstance(allowed_msg, (bytes, bytearray)) and len(allowed_msg) > 0 + + # Denied pre should have NO events (StopIteration immediately) + denied_iter = wesHab.db.clonePreIter(pre=palDenied.pre) + with pytest.raises(StopIteration): + next(denied_iter) + return True class MockServerTls: def __init__(self, certify, keypath, certpath, cafilepath, port): @@ -271,27 +348,69 @@ def __init__(self, certify, keypath, certpath, cafilepath, port): class MockHttpServer: - def __init__(self, port, app, servant=None): + def __init__(self, host, port, app, servant=None): self.servant = servant def test_createHttpServer(monkeypatch): + host = "0.0.0.0" port = 5632 app = falcon.App() - server = indirecting.createHttpServer(port, app) + server = indirecting.createHttpServer(host, port, app) assert isinstance(server, http.Server) monkeypatch.setattr(hio.core.tcp, 'ServerTls', MockServerTls) monkeypatch.setattr(hio.core.http, 'Server', MockHttpServer) - server = indirecting.createHttpServer(port, app, keypath='keypath', certpath='certpath', cafilepath='cafilepath') + server = indirecting.createHttpServer(host, port, app, keypath='keypath', certpath='certpath', cafilepath='cafilepath') assert isinstance(server, MockHttpServer) assert isinstance(server.servant, MockServerTls) +def test_metrics_end(): + """Test MetricsEnd returns Prometheus format metrics""" + with habbing.openHby(name="test", salt=core.Salter(raw=b'0123456789abcdef').qb64, temp=True) as hby: + reger = viring.Reger(name=hby.name, db=hby.db, temp=True) + + app = falcon.App() + metricsEnd = EscrowEnd(hby=hby, reger=reger) + app.add_route("/metrics", metricsEnd) + + client = testing.TestClient(app) + + # Test GET /metrics + res = client.simulate_get("/metrics") + assert res.status_code == 200 + assert "text/plain" in res.headers['Content-Type'] + + # Verify Prometheus format + body = res.text + assert "# HELP keri_escrow_count" in body + assert "# TYPE keri_escrow_count gauge" in body + + # Verify KEL escrow metrics present + assert 'keri_escrow_count{type="out_of_order_events",layer="kel"}' in body + assert 'keri_escrow_count{type="partially_witnessed_events",layer="kel"}' in body + assert 'keri_escrow_count{type="unverified_receipts",layer="kel"}' in body + + # Verify TEL escrow metrics present + assert 'keri_escrow_count{type="out_of_order",layer="tel"}' in body + assert 'keri_escrow_count{type="missing_registry",layer="tel"}' in body + + # Verify registry escrow metrics present + assert 'keri_escrow_count{type="registry_missing_anchor",layer="registry"}' in body + + # All counts should be 0 for empty db + lines = [l for l in body.split('\n') if l and not l.startswith('#')] + for line in lines: + assert line.endswith(' 0'), f"Expected count 0, got: {line}" + + reger.close() if __name__ == "__main__": test_mailbox_iter() test_qrymailbox_iter() + test_wit_query_ends() + test_metrics_end() diff --git a/tests/app/test_keeper_mapsize.py b/tests/app/test_keeper_mapsize.py new file mode 100644 index 000000000..318c4d8ec --- /dev/null +++ b/tests/app/test_keeper_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_keeper_mapsize module + +""" +import os +import pytest + + +def test_keeper_specific_env_var(): + from keri.app import keeping + + os.environ['KERI_KEEPER_MAP_SIZE'] = '150000000' + + try: + keeper = keeping.Keeper(name='test_keeper', temp=True) + assert keeper.MapSize == 150000000 + keeper.close() + finally: + os.environ.pop('KERI_KEEPER_MAP_SIZE', None) + + +def test_keeper_general_env_var_fallback(): + from keri.app import keeping + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + keeper = keeping.Keeper(name='test_keeper_general', temp=True) + assert keeper.MapSize == 250000000 + keeper.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_keeper_specific_takes_precedence(): + from keri.app import keeping + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_KEEPER_MAP_SIZE'] = '200000000' + + try: + keeper = keeping.Keeper(name='test_keeper_precedence', temp=True) + assert keeper.MapSize == 200000000 + keeper.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_KEEPER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_keeper_specific_env_var() + test_keeper_general_env_var_fallback() + test_keeper_specific_takes_precedence() diff --git a/tests/app/test_keeping.py b/tests/app/test_keeping.py index 0912413c3..f15605746 100644 --- a/tests/app/test_keeping.py +++ b/tests/app/test_keeping.py @@ -16,9 +16,17 @@ from hio.base import doing +from keri import kering +from keri.app.keeping import Keeper, KERIKeeperMapSizeKey +from keri.db.basing import KERIBaserMapSizeKey +from keri.db.dbing import LMDBer, KERILMDBMapSizeKey from keri.help import helping -from keri.core import coring -from keri.core.coring import IdrDex + +from keri import core + +from keri.core import coring, indexing +from keri.core.indexing import IdrDex + from keri.app import keeping @@ -389,20 +397,20 @@ def test_keeper(): key = b'tier' assert keeper.gbls.get(key) == None assert keeper.gbls.rem(key) == False - assert keeper.gbls.put(key, val=coring.Tiers.low) == True - assert keeper.gbls.get(key) == coring.Tiers.low - assert keeper.gbls.put(key, val=coring.Tiers.med) == False - assert keeper.gbls.get(key) == coring.Tiers.low - assert keeper.gbls.pin(key, val=coring.Tiers.med) == True - assert keeper.gbls.get(key) == coring.Tiers.med + assert keeper.gbls.put(key, val=core.Tiers.low) == True + assert keeper.gbls.get(key) == core.Tiers.low + assert keeper.gbls.put(key, val=core.Tiers.med) == False + assert keeper.gbls.get(key) == core.Tiers.low + assert keeper.gbls.pin(key, val=core.Tiers.med) == True + assert keeper.gbls.get(key) == core.Tiers.med assert keeper.gbls.rem(key) == True assert keeper.gbls.get(key) == None # test .pris sub db methods key = puba - signera = coring.Signer(qb64b=pria) + signera = core.Signer(qb64b=pria) assert signera.qb64b == pria - signerb = coring.Signer(qb64b=prib) + signerb = core.Signer(qb64b=prib) assert signerb.qb64b == prib assert keeper.pris.get(key) == None assert keeper.pris.rem(key) == False @@ -601,7 +609,7 @@ def test_creator(): signers = creator.create() assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519 assert signer.verfer.code not in coring.NonTransDex @@ -609,7 +617,7 @@ def test_creator(): signers = creator.create(count=2, transferable=False) assert len(signers) == 2 for signer in signers: - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519N assert signer.verfer.code in coring.NonTransDex @@ -617,7 +625,7 @@ def test_creator(): creator = keeping.SaltyCreator() assert isinstance(creator, keeping.SaltyCreator) assert isinstance(creator, keeping.Creator) - assert isinstance(creator.salter, coring.Salter) + assert isinstance(creator.salter, core.Salter) assert creator.salter.code == coring.MtrDex.Salt_128 assert creator.salt == creator.salter.qb64 assert creator.stem == '' @@ -625,7 +633,7 @@ def test_creator(): signers = creator.create() assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519 assert signer.verfer.code not in coring.NonTransDex @@ -633,25 +641,25 @@ def test_creator(): signers = creator.create(count=2, transferable=False) assert len(signers) == 2 for signer in signers: - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.verfer.code == coring.MtrDex.Ed25519N assert signer.verfer.code in coring.NonTransDex raw = b'0123456789abcdef' - salt = coring.Salter(raw=raw).qb64 + salt = core.Salter(raw=raw).qb64 assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' creator = keeping.SaltyCreator(salt=salt) assert isinstance(creator, keeping.SaltyCreator) assert isinstance(creator, keeping.Creator) - assert isinstance(creator.salter, coring.Salter) + assert isinstance(creator.salter, core.Salter) assert creator.salter.code == coring.MtrDex.Salt_128 assert creator.salter.raw == raw assert creator.salter.qb64 == salt signers = creator.create() assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.qb64 == 'APMJe0lwOpwnX9PkvX1mh26vlzGYl6RWgWGclc8CAQJ9' assert signer.verfer.code == coring.MtrDex.Ed25519 @@ -661,7 +669,7 @@ def test_creator(): signers = creator.create(count=1, transferable=False, temp=True) assert len(signers) == 1 signer = signers[0] - assert isinstance(signer, coring.Signer) + assert isinstance(signer, core.Signer) assert signer.code == coring.MtrDex.Ed25519_Seed assert signer.qb64 == 'AMGrAM0noxLpRteO9mxGT-yzYSrKFwJMuNI4KlmSk26e' assert signer.verfer.code == coring.MtrDex.Ed25519N @@ -692,7 +700,7 @@ def test_manager(): assert not manager.ks.opened raw = b'0123456789abcdef' - salt = coring.Salter(raw=raw).qb64 + salt = core.Salter(raw=raw).qb64 stem = "red" assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' @@ -708,14 +716,14 @@ def test_manager(): with keeping.openKS() as keeper: - with pytest.raises(ValueError): + with pytest.raises(kering.ConversionError): #test invalid qb64 of Salt manager = keeping.Manager(ks=keeper, salt='0AzwMTIzNDU2Nzg5YWJjZGVm') manager = keeping.Manager(ks=keeper, salt=salt) assert manager.ks.opened assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low assert manager.salt == salt assert manager.aeid == "" assert manager.seed == "" @@ -736,7 +744,7 @@ def test_manager(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == [] @@ -773,7 +781,7 @@ def test_manager(): psigers = manager.sign(ser=ser, pubs=ps.new.pubs) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) vsigers = manager.sign(ser=ser, verfers=verfers) psigs = [siger.qb64 for siger in psigers] vsigs = [siger.qb64 for siger in vsigers] @@ -786,7 +794,7 @@ def test_manager(): # Test with pubs list psigers = manager.sign(ser=ser, pubs=ps.new.pubs, indices=indices) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] psigs = [siger.qb64 for siger in psigers] assert psigs == ['ADAa70b4QnTOtGOsMqcezMtVzCFuRJHGeIMkWYHZ5ZxGIXM0XDVAzkYdCeadfPfzlKC6dkfiwuJ0IzLOElaanUgH'] @@ -794,7 +802,7 @@ def test_manager(): # Test with verfers list vsigers = manager.sign(ser=ser, verfers=verfers, indices=indices) for siger in vsigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] vsigs = [siger.qb64 for siger in vsigers] assert vsigs == psigs @@ -819,7 +827,7 @@ def test_manager(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == ['DFRtyHAjSuJaRX6TDPva35GN11VHAruaOXMc79ZYDKsT'] @@ -952,7 +960,7 @@ def test_manager(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == stem == 'red' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == [] @@ -1019,7 +1027,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 6 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 0 ipre, verferies = manager.ingest(secrecies=secrecies) # use default iridx assert ipre == 'DNsGfyf7JArtQgioD7BdVwRulGAsQk5REIKSTjFJqE0a' @@ -1093,7 +1101,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 7 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 3 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx) @@ -1167,7 +1175,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 8 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 7 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx) @@ -1249,7 +1257,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 9 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 0 ipre, verferies = manager.ingest(secrecies=secrecies, ncount=3) # default iridx @@ -1343,7 +1351,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 10 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 1 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx, ncount=3) assert ipre == 'DHc6ZvVpAjLq_digYYZMhq0OlEnnbCrgSvcxPJQY_oAE' @@ -1436,7 +1444,7 @@ def test_manager(): assert manager.aeid == '' assert manager.pidx == 11 assert manager.salt == salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low iridx = 3 ipre, verferies = manager.ingest(secrecies=secrecies, iridx=iridx, ncount=3) assert ipre == 'DO1mU48dTzyFtHjqH744gl0QoEIMxphSC4qbgMoEuyq7' @@ -1516,7 +1524,7 @@ def test_manager_with_aeid(): """ # rawsalt =pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) rawsalt = b'0123456789abcdef' - salter = coring.Salter(raw=rawsalt) + salter = core.Salter(raw=rawsalt) salt = salter.qb64 assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' stem = "blue" @@ -1524,25 +1532,25 @@ def test_manager_with_aeid(): # cryptseed0 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed0 = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner0 = coring.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, + cryptsigner0 = core.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, transferable=False) seed0 = cryptsigner0.qb64 aeid0 = cryptsigner0.verfer.qb64 assert aeid0 == 'BCa7mK96FwxkU0TdF54Yqg3qBDXUWpOhQ_Mtr7E77yZB' - decrypter0 = coring.Decrypter(seed=seed0) - encrypter0 = coring.Encrypter(verkey=aeid0) + decrypter0 = core.Decrypter(seed=seed0) + encrypter0 = core.Encrypter(verkey=aeid0) assert encrypter0.verifySeed(seed=seed0) # cryptseed1 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed1 = (b"\x89\xfe{\xd9'\xa7\xb3\x89#\x19\xbec\xee\xed\xc0\xf9\x97\xd0\x8f9\x1dyNI" b'I\x98\xbd\xa4\xf6\xfe\xbb\x03') - cryptsigner1 = coring.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, + cryptsigner1 = core.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, transferable=False) seed1 = cryptsigner1.qb64 aeid1 = cryptsigner1.verfer.qb64 assert aeid1 == 'BEcOrMrG_7r_NWaLl6h8UJapwIfQWIkjrIPXkCZm2fFM' - decrypter1 = coring.Decrypter(seed=seed1) - encrypter1 = coring.Encrypter(verkey=aeid1) + decrypter1 = core.Decrypter(seed=seed1) + encrypter1 = core.Encrypter(verkey=aeid1) assert encrypter1.verifySeed(seed=seed1) # something to sign doesn't matter what for testing purposes @@ -1570,8 +1578,8 @@ def test_manager_with_aeid(): assert manager.algo == keeping.Algos.salty assert manager.salt == salt # encrypted on disk but property decrypts if seed assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low - saltCipher0 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + assert manager.tier == core.Tiers.low + saltCipher0 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert saltCipher0.decrypt(seed=seed0).qb64 == salt # salty algorithm incept @@ -1586,9 +1594,9 @@ def test_manager_with_aeid(): pp = manager.ks.prms.get(spre) assert pp.pidx == 0 assert pp.algo == keeping.Algos.salty - assert manager.decrypter.decrypt(ser=pp.salt).qb64 == salt + assert manager.decrypter.decrypt(qb64=pp.salt).qb64 == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == [] @@ -1625,7 +1633,7 @@ def test_manager_with_aeid(): psigers = manager.sign(ser=ser, pubs=ps.new.pubs) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) vsigers = manager.sign(ser=ser, verfers=verfers) psigs = [siger.qb64 for siger in psigers] vsigs = [siger.qb64 for siger in vsigers] @@ -1638,7 +1646,7 @@ def test_manager_with_aeid(): # Test with pubs list psigers = manager.sign(ser=ser, pubs=ps.new.pubs, indices=indices) for siger in psigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] psigs = [siger.qb64 for siger in psigers] assert psigs == ['ADAa70b4QnTOtGOsMqcezMtVzCFuRJHGeIMkWYHZ5ZxGIXM0XDVAzkYdCeadfPfzlKC6dkfiwuJ0IzLOElaanUgH'] @@ -1646,7 +1654,7 @@ def test_manager_with_aeid(): # Test with verfers list vsigers = manager.sign(ser=ser, verfers=verfers, indices=indices) for siger in vsigers: - assert isinstance(siger, coring.Siger) + assert isinstance(siger, indexing.Siger) assert psigers[0].index == indices[0] vsigs = [siger.qb64 for siger in vsigers] assert vsigs == psigs @@ -1669,9 +1677,9 @@ def test_manager_with_aeid(): pp = manager.ks.prms.get(spre) assert pp.pidx == 0 assert pp.algo == keeping.Algos.salty - assert manager.decrypter.decrypt(ser=pp.salt).qb64 == salt + assert manager.decrypter.decrypt(qb64=pp.salt).qb64 == salt assert pp.stem == '' - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(spre) assert ps.old.pubs == ['DFRtyHAjSuJaRX6TDPva35GN11VHAruaOXMc79ZYDKsT'] @@ -1701,8 +1709,8 @@ def test_manager_with_aeid(): assert manager.algo == keeping.Algos.salty assert manager.salt == salt assert manager.pidx == 1 - assert manager.tier == coring.Tiers.low - saltCipher1 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + assert manager.tier == core.Tiers.low + saltCipher1 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert saltCipher1.decrypt(seed=seed1).qb64 == salt assert not saltCipher0.qb64 == saltCipher1.qb64 # old cipher different @@ -1800,7 +1808,7 @@ def test_manager_sign_dual_indices(): When ondices is not provided then all sigers .ondex is None. """ raw = b'0123456789abcdef' - salt = coring.Salter(raw=raw).qb64 + salt = core.Salter(raw=raw).qb64 # the particular serialization does not matter for test purposes ser = (b"See ya later Alligator. In a while Crocodile. " @@ -1812,7 +1820,7 @@ def test_manager_sign_dual_indices(): manager = keeping.Manager(ks=keeper, salt=salt) assert manager.ks.opened assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low + assert manager.tier == core.Tiers.low assert manager.salt == salt assert manager.aeid == "" assert manager.seed == "" @@ -1840,7 +1848,7 @@ def test_manager_sign_dual_indices(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == stem - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(pre) assert len(ps.old.pubs) == 0 @@ -1949,7 +1957,7 @@ def test_manager_sign_dual_indices(): assert pp.algo == keeping.Algos.salty assert pp.salt == salt assert pp.stem == stem - assert pp.tier == coring.Tiers.low + assert pp.tier == core.Tiers.low ps = manager.ks.sits.get(pre) assert len(ps.old.pubs) == ocount @@ -1992,5 +2000,44 @@ def test_manager_sign_dual_indices(): assert not manager.ks.opened """End Test""" +def test_keeper_db_size_set_from_env_var(): + # Clear environment before test + if KERIBaserMapSizeKey in os.environ: + os.environ.pop(KERILMDBMapSizeKey) + if KERIKeeperMapSizeKey in os.environ: + os.environ.pop(KERIKeeperMapSizeKey) + + new_map_size = 10737418240 + + # Default map size works + kpr = Keeper(reopen=True) + assert kpr.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert kpr.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + kpr.close() + + # Specific map size works + os.environ[KERIKeeperMapSizeKey] = f"{new_map_size}" + + kpr = Keeper(reopen=True) + assert kpr.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERIKeeperMapSizeKey) + kpr.close() + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERILMDBMapSizeKey] = f"{baser_map_size}" + + kpr = Keeper(reopen=True) + assert kpr.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + kpr.close() + + # Bad map size throws + os.environ[KERIKeeperMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + kpr = Keeper(reopen=True) + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERILMDBMapSizeKey) + os.environ.pop(KERIKeeperMapSizeKey) + if __name__ == "__main__": test_manager_sign_dual_indices() diff --git a/tests/app/test_mailboxer_mapsize.py b/tests/app/test_mailboxer_mapsize.py new file mode 100644 index 000000000..65d3115be --- /dev/null +++ b/tests/app/test_mailboxer_mapsize.py @@ -0,0 +1,55 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_mailboxer_mapsize module + +""" +import os +import pytest + + +def test_mailboxer_specific_env_var(): + from keri.app import storing + + os.environ['KERI_MAILBOXER_MAP_SIZE'] = '150000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer', temp=True) + assert mbx.MapSize == 150000000 + mbx.close() + finally: + os.environ.pop('KERI_MAILBOXER_MAP_SIZE', None) + + +def test_mailboxer_general_env_var_fallback(): + from keri.app import storing + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer_general', temp=True) + assert mbx.MapSize == 250000000 + mbx.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_mailboxer_specific_takes_precedence(): + """Test that KERI_MAILBOXER_MAP_SIZE takes precedence over KERI_LMDB_MAP_SIZE""" + from keri.app import storing + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_MAILBOXER_MAP_SIZE'] = '200000000' + + try: + mbx = storing.Mailboxer(name='test_mailboxer_precedence', temp=True) + assert mbx.MapSize == 200000000 + mbx.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_MAILBOXER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_mailboxer_specific_env_var() + test_mailboxer_general_env_var_fallback() + test_mailboxer_specific_takes_precedence() diff --git a/tests/app/test_noter_mapsize.py b/tests/app/test_noter_mapsize.py new file mode 100644 index 000000000..4748092a4 --- /dev/null +++ b/tests/app/test_noter_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_noter_mapsize module + +""" +import os +import pytest + + +def test_noter_specific_env_var(): + from keri.app import notifying + + os.environ['KERI_NOTER_MAP_SIZE'] = '150000000' + + try: + noter = notifying.Noter(name='test_noter', temp=True) + assert noter.MapSize == 150000000 + noter.close() + finally: + os.environ.pop('KERI_NOTER_MAP_SIZE', None) + + +def test_noter_general_env_var_fallback(): + from keri.app import notifying + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + noter = notifying.Noter(name='test_noter_general', temp=True) + assert noter.MapSize == 250000000 + noter.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_noter_specific_takes_precedence(): + from keri.app import notifying + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_NOTER_MAP_SIZE'] = '200000000' + + try: + noter = notifying.Noter(name='test_noter_precedence', temp=True) + assert noter.MapSize == 200000000 + noter.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_NOTER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_noter_specific_env_var() + test_noter_general_env_var_fallback() + test_noter_specific_takes_precedence() diff --git a/tests/app/test_notifying.py b/tests/app/test_notifying.py index b02bd91c2..1ebe40928 100644 --- a/tests/app/test_notifying.py +++ b/tests/app/test_notifying.py @@ -4,12 +4,16 @@ """ import datetime +import os import pytest from keri.app import notifying, habbing +from keri.app.notifying import Noter, KERINoterMapSizeKey from keri.core import coring from keri.db import dbing +from keri.db.basing import KERIBaserMapSizeKey +from keri.db.dbing import LMDBer, KERILMDBMapSizeKey from keri.help import helping @@ -105,7 +109,7 @@ def test_dictersuber(): assert res[2].attrs['a'] == 3 -def test_noter(): +def test_noter(mockHelpingNowUTC): noter = notifying.Noter() assert noter.path.endswith("/not/not") noter.reopen() @@ -137,12 +141,14 @@ def test_noter(): notes = noter.getNotes(start=0) assert len(notes) == 0 - dt = datetime.datetime.now() - note = notifying.notice(attrs=dict(a=1)) + note = notifying.notice(attrs=dict(a=1), + dt=helping.fromIso8601("2022-07-08T15:01:05.453632")) assert noter.add(note, cig) is True - note = notifying.notice(attrs=dict(a=2)) + note = notifying.notice(attrs=dict(a=2), + dt=helping.fromIso8601("2022-07-08T15:01:06.453632")) assert noter.add(note, cig) is True - note = notifying.notice(attrs=dict(a=3)) + note = notifying.notice(attrs=dict(a=3), + dt=helping.fromIso8601("2022-07-08T15:01:07.453632")) assert noter.add(note, cig) is True res = [] @@ -164,12 +170,13 @@ def test_noter(): res.append(note) assert len(res) == 5 + assert res[0][0].datetime == "2021-01-01T00:00:00.000000+00:00" cnt = noter.getNoteCnt() assert cnt == 13 -def test_notifier(): +def test_notifier(mockHelpingNowUTC): with habbing.openHby(name="test") as hby: notifier = notifying.Notifier(hby=hby) assert notifier.signaler is not None @@ -200,7 +207,7 @@ def test_notifier(): assert notifier.rem(note.rid) is True assert notifier.getNotes() == [] - dt = datetime.datetime.now() + dt = helping.nowIso8601() assert notifier.add(attrs=dict(a=1)) is True assert notifier.add(attrs=dict(a=2)) is True assert notifier.add(attrs=dict(a=3)) is True @@ -208,6 +215,8 @@ def test_notifier(): notes = notifier.getNotes() assert len(notes) == 3 + assert notes[2].datetime == "2021-01-01T00:00:00.000000+00:00" + payload = dict(a=1, b=2, c=3) dt = helping.fromIso8601("2022-07-08T15:01:05.453632") cig = coring.Cigar(qb64="AABr1EJXI1sTuI51TXo4F1JjxIJzwPeCxa-Cfbboi7F4Y4GatPEvK629M7G_5c86_Ssvwg8POZWNMV-WreVqBECw") @@ -216,3 +225,41 @@ def test_notifier(): assert notifier.mar(note.rid) is False assert notifier.rem(note.rid) is True + +def test_noter_db_size_set_from_env_var(): + # Clear environment before test + if KERILMDBMapSizeKey in os.environ: + os.environ.pop(KERILMDBMapSizeKey) + if KERINoterMapSizeKey in os.environ: + os.environ.pop(KERINoterMapSizeKey) + + new_map_size = 10737418240 + # Default map size works + noter = Noter() + assert noter.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert noter.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + noter.close() + + # Specific map size works + os.environ[KERINoterMapSizeKey] = f"{new_map_size}" + + noter = Noter() + assert noter.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERINoterMapSizeKey) + noter.close() + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERILMDBMapSizeKey] = f"{baser_map_size}" + + noter = Noter() + assert noter.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + noter.close() + + # bad map size throws + os.environ[KERINoterMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + Noter() + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERILMDBMapSizeKey) + os.environ.pop(KERINoterMapSizeKey) diff --git a/tests/app/test_oobiing.py b/tests/app/test_oobiing.py index 4963e1e8a..b9280f7d9 100644 --- a/tests/app/test_oobiing.py +++ b/tests/app/test_oobiing.py @@ -3,29 +3,25 @@ tests.app.test_multisig module """ -import json import falcon -from falcon import testing from hio.base import doing +from hio.core import http import keri -from hio.core import http +from keri import help, kering, core from keri.app import habbing, oobiing, notifying -from keri.core import coring, parsing, serdering +from keri.core import serdering, eventing, parsing, routing from keri.db import basing from keri.end import ending from keri.help import helping -from keri import help, kering from keri.peer import exchanging -from tests.app import openMultiSig - def test_oobi_share(mockHelpingNowUTC): oobi = "http://127.0.0.1:5642/oobi/Egw3N07Ajdkjvv4LB2Mhx2qxl6TOCFdWNJU6cYR_ImFg/witness" \ "/BGKVzj4ve0VSd8z_AmvhLg4lqcC_9WYX90k03q-R_Ydo?name=Phil" - with habbing.openHab(name="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): exc = exchanging.Exchanger(hby=hby, handlers=[]) notifier = notifying.Notifier(hby=hby) @@ -55,21 +51,22 @@ def test_oobi_share(mockHelpingNowUTC): exn, atc = oobiing.oobiRequestExn(hab=hab, dest="EO2kxXW0jifQmuPevqg6Zpi3vE-WYoj65i_XhpruWtOg", oobi="http://127.0.0.1/oobi") - assert exn.ked == {'v': 'KERI10JSON000136_', 't': 'exn', 'd': 'EII7EvdWFqv0jkjRv10t01zAUcRYbjVhZ_yo3VPZEbpS', + assert exn.ked == {'a': {'dest': 'EO2kxXW0jifQmuPevqg6Zpi3vE-WYoj65i_XhpruWtOg', + 'oobi': 'http://127.0.0.1/oobi'}, + 'd': 'EII7EvdWFqv0jkjRv10t01zAUcRYbjVhZ_yo3VPZEbpS', + 'dt': '2021-01-01T00:00:00.000000+00:00', + 'e': {}, 'i': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', - 'rp': '', 'p': '', - 'dt': '2021-01-01T00:00:00.000000+00:00', - 'r': '/oobis', 'q': {}, - 'a': { - 'dest': 'EO2kxXW0jifQmuPevqg6Zpi3vE-WYoj65i_XhpruWtOg', - 'oobi': 'http://127.0.0.1/oobi'}, - 'e': {} - } - assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg' - b'-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' - b'-AABAABdw3eSw_7BW2o3z1ufxxs1CPgX1TgtJzn-MxvMjLYTidUd8KSxNKbPU9M3A4orYJDMGMIzhabHJmKA4ZIGbcgK') + 'r': '/oobis', + 'rp': '', + 't': 'exn', + 'v': 'KERI10JSON000136_'} + assert atc == (b'-FABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAA' + b'AAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAABdw3eS' + b'w_7BW2o3z1ufxxs1CPgX1TgtJzn-MxvMjLYTidUd8KSxNKbPU9M3A4orYJDMGMIz' + b'habHJmKA4ZIGbcgK') def test_oobiery(): @@ -133,6 +130,68 @@ def test_oobiery(): """Done Test""" +def test_introduce(mockHelpingNowUTC): + raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' + salt = core.Salter(raw=raw).qb64 + assert salt == '0AAFqo8tU5rp-lWcApybCEh1' + # makHab uses stem=name to make different names have differnt AID pre + with (habbing.openHby(name="wat", base="test", salt=salt) as watHby, + habbing.openHby(name="wit", base="test", salt=salt) as witHby): + # setup Wes's habitat nontrans + watHab = watHby.makeHab(name='wes', isith="1", icount=1, transferable=False) + assert not watHab.kever.prefixer.transferable + assert watHab.pre == "BBVDlgWic_rAf-m_v7vz_VvIYAUPErvZgLTfXGNrFRom" + watKvy = eventing.Kevery(db=watHab.db, lax=False, local=False) + watPsr = parsing.Parser(kvy=watKvy) + + # setup Wok's habitat nontrans + witHab = witHby.makeHab(name='wok', isith="1", icount=1, transferable=False) + assert not witHab.kever.prefixer.transferable + assert witHab.pre == "BKVb58uITf48YoMPz8SBOTVwLgTO9BY4oEXRPoYIOErX" + witKvy = eventing.Kevery(db=witHab.db, lax=False, local=False) + + rtr = routing.Router() + rvy = routing.Revery(db=witHby.db, rtr=rtr) + oobiing.Oobiery(hby=witHby, rvy=rvy) + witPsr = parsing.Parser(kvy=witKvy, rvy=rvy) + assert witHby.db.oobis.cntAll() == 0 + + oobi = f"https://localhost:8989/oobi/{watHab.pre}/controller" + data = dict( + cid=watHab.pre, + oobi=oobi + ) + + msg = watHab.reply(route="/introduce", data=data) + assert msg == (b'{"v":"KERI10JSON000127_","t":"rpy","d":"EPEU3V7e2d2mhMWVFDS-oC9z' + b'Q8DX8t6ELkhINIaYGFNZ","dt":"2021-01-01T00:00:00.000000+00:00","r' + b'":"/introduce","a":{"cid":"BBVDlgWic_rAf-m_v7vz_VvIYAUPErvZgLTfX' + b'GNrFRom","oobi":"https://localhost:8989/oobi/BBVDlgWic_rAf-m_v7v' + b'z_VvIYAUPErvZgLTfXGNrFRom/controller"}}-VAi-CABBBVDlgWic_rAf-m_v' + b'7vz_VvIYAUPErvZgLTfXGNrFRom0BBqF8yHDeXpzDUNIOsDBGezNBdgHafmOYbQ7' + b'qw0R5t89FbLA26RwaA3NF9-dU0JpbNuJs7jiEBYbSGeDkbBDDEM') + + witPsr.parseOne(ims=msg) + assert witHby.db.oobis.cntAll() == 1 + obr = witHby.db.oobis.get(keys=(oobi,)) + assert obr.cid == watHab.pre + + # Send one missing fields + data = dict(cid=watHab.pre) + msg = watHab.reply(route="/introduce", data=data) + witPsr.parseOne(ims=msg) + assert witHby.db.oobis.cntAll() == 1 # Still one because of the missing 'oobi' field + + # Send one bad scheme + data = dict(cid=watHab.pre, oobi="ftp://localhost") + msg = watHab.reply(route="/introduce", data=data) + witPsr.parseOne(ims=msg) + assert witHby.db.oobis.cntAll() == 1 # Still one because of the missing 'oobi' field + + + + + class MOOBIEnd: """ Test endpoint returning a static MOOBI """ def __init__(self, hab, url): diff --git a/tests/app/test_querying.py b/tests/app/test_querying.py index d723eaa94..24b618f70 100644 --- a/tests/app/test_querying.py +++ b/tests/app/test_querying.py @@ -3,14 +3,22 @@ keri.app.querying module """ +import logging + from hio.base import doing +from keri import help from keri.app import habbing from keri.app.querying import QueryDoer, KeyStateNoticer, LogQuerier, SeqNoQuerier, AnchorQuerier -from keri.core import parsing, eventing +from keri.core import parsing, eventing, serdering +from keri.db.dbing import dgKey + + +logger = help.ogler.getLogger() def test_querying(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks with habbing.openHby() as hby, \ habbing.openHby() as hby1: inqHab = hby.makeHab(name="inquisitor") @@ -37,7 +45,7 @@ def test_querying(): assert msg["src"] == inqHab.pre assert msg["pre"] == subHab.pre assert msg["r"] == "ksn" - assert msg["q"] == {'s': '0'} + assert msg["q"] == {'fn': '0', 's': '0'} assert msg["wits"] is None doist.recur(deeds=deeds) @@ -108,6 +116,12 @@ def test_querying(): doist.recur(deeds=deeds) assert len(sdoer.witq.msgs) == 1 + sdoer = SeqNoQuerier(hby=hby, hab=inqHab, pre=subHab.pre, fn=2, sn=4) + assert len(sdoer.witq.msgs) == 1 + msg = sdoer.witq.msgs.pull() + query = msg['q'] + assert query == {'fn': '2', 's': '4'} + # Test with originally unknown AID sdoer = SeqNoQuerier(hby=hby, hab=inqHab, pre="ExxCHAI9bkl50F5SCKl2AWQbFGKeJtz0uxM2diTMxMQA", sn=1) assert len(sdoer.witq.msgs) == 1 @@ -131,7 +145,8 @@ def test_querying(): assert len(sdoer.witq.msgs) == 1 # Test with originally unknown AID - adoer = AnchorQuerier(hby=hby, hab=inqHab, pre="ExxCHAI9bkl50F5SCKl2AWQbFGKeJtz0uxM2diTMxMQA", anchor={'s': '5'}) + adoer = AnchorQuerier(hby=hby, hab=inqHab, pre="ExxCHAI9bkl50F5SCKl2AWQbFGKeJtz0uxM2diTMxMQA", + anchor={'s': '5'}) assert len(adoer.witq.msgs) == 1 tock = 0.03125 @@ -141,3 +156,26 @@ def test_querying(): doist.recur(deeds=deeds) assert len(adoer.witq.msgs) == 1 +def test_query_not_found_escrow(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + with habbing.openHby() as inqHby, \ + habbing.openHby() as subHby: + inqHab = inqHby.makeHab(name="inquisitor") + subHab = subHby.makeHab(name="subject") + + psr = parsing.Parser() + + icp = inqHab.makeOwnInception() + subKvy = eventing.Kevery(db=subHab.db, lax=False, local=False) + psr.parseOne(ims=bytearray(icp), kvy=subKvy) + assert inqHab.pre in subHab.kevers + + qry = inqHab.query(subHab.pre, route="/foo", src=inqHab.pre) + serder = serdering.SerderKERI(raw=qry) + dgkey = dgKey(inqHab.pre, serder.saidb) + + subHab.db.putEvt(dgkey, serder.raw) + subHab.db.qnfs.add(keys=(inqHab.pre, serder.said), val=serder.saidb) + + subHab.kvy.processQueryNotFound() + assert subHab.db.qnfs.get(dgkey) == [] \ No newline at end of file diff --git a/tests/app/test_signaling.py b/tests/app/test_signaling.py index 9010d9e89..fb8f85580 100644 --- a/tests/app/test_signaling.py +++ b/tests/app/test_signaling.py @@ -79,7 +79,7 @@ def test_signaler(): signaler.push(attrs=dict(a=1), topic="/m") signaler.push(attrs=dict(a=2), topic="/m", ckey="abc") signaler.push(attrs=dict(a=3), topic="/m") - now = datetime.datetime.now() - datetime.timedelta(minutes=11) + now = helping.nowUTC() - datetime.timedelta(minutes=11) signaler.push(attrs=dict(a=4), topic="/m", ckey="abc", dt=now) assert len(signaler.signals) == 3 diff --git a/tests/app/test_signify.py b/tests/app/test_signify.py index 4afaa4647..205d44bf9 100644 --- a/tests/app/test_signify.py +++ b/tests/app/test_signify.py @@ -6,16 +6,20 @@ import pytest from keri import kering + +from keri import core +from keri.core import coring, eventing + from keri.app import habbing from keri.app.keeping import SaltyCreator -from keri.core import coring, eventing + def test_remote_salty_hab(): name = "test" - tier = coring.Tiers.low + tier = core.Tiers.low raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = coring.Salter(raw=raw, tier=tier) + salter = core.Salter(raw=raw, tier=tier) with habbing.openHby(name="remoteSalty") as remote, \ habbing.openHby(name="local", salt=salter.qb64, temp=True, tier=tier) as local: @@ -71,7 +75,7 @@ def test_remote_salty_hab(): assert [verfer.qb64 for verfer in kever.verfers] == keys assert [diger.qb64 for diger in kever.ndigers] == nxt - habord = remote.db.habs.get(name) + habord = remote.db.habs.get(hab.pre) assert habord.hid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" assert habord.sid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" @@ -113,7 +117,7 @@ def test_remote_salty_hab(): assert [verfer.qb64 for verfer in kever.verfers] == keys1 assert [diger.qb64 for diger in kever.ndigers] == nxt1 - habord = remote.db.habs.get(name) + habord = remote.db.habs.get(hab.pre) assert habord.hid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" assert habord.sid == "EHeU-ldGfJhxceV9BTq38HdFUoasoWEcYATiyZCcDH7N" diff --git a/tests/app/test_signing.py b/tests/app/test_signing.py index 3561d2201..9a132990a 100644 --- a/tests/app/test_signing.py +++ b/tests/app/test_signing.py @@ -3,11 +3,13 @@ tests.app.signing module """ +from keri import kering, core +from keri.core import coring, parsing, eventing +from keri.core.eventing import SealEvent from keri.app import habbing, configing, keeping from keri.app import signing -from keri.core import coring, parsing, eventing -from keri.core.eventing import SealEvent + from keri.db import basing from keri.vc import proving from keri.vdr import verifying, credentialing @@ -44,89 +46,90 @@ def test_sad_signature(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): # Sign with non-transferable identifier, default to entire SAD sig0 = signing.ratify(wanHab, cred) - assert sig0 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSKl' - b'DJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri":' - b'"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLBSMZ' - b'_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg39j1b' - b'7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFRD9dTb' - b'4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7xMwsxUe' - b'lUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36","pers' - b'onal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-","n":"Q8r' - b'NaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalName":"Anna ' - b'Jones","engagementContextRole":"Project Manager"}},"e":[{"qualifie' - b'dvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU' - b'5sHYTGFD"}]}-JAB6AABAAA--CABBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBw' - b'bAqteP0BApbos85syKE_VgfuNTtVRYkAlw5fwb_4ZWN-V-FFO_MrSGt71luX0rt-9e' - b'hNZFPHV1EuPc1YDQvZJ1XqPumewN') + assert sig0 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-JAB6AABAAA--CABBEByXTm1zJZ1oI' + b'uvnad6hppC3iBrUmQLb0a7iXSaRK4p0BCCKuUzhtasI04dqYQY4x7AqHMLP7Cbn2' + b'nqKFrR34_4zjJvT3HARSLhL_iUOSyoHMVv2-9mgtWuuW0oF4XS3n0C') # sign with non-trans identifier with a specific set of paths sig1 = signing.ratify(wanHab, cred, paths=paths) - assert sig1 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSK' - b'lDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri' - b'":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLB' - b'SMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg3' - b'9j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFR' - b'D9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7x' - b'MwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36' - b'","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-",' - b'"n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalNam' - b'e":"Anna Jones","engagementContextRole":"Project Manager"}},"e":[' - b'{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC' - b'0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--CADBAbSj3jfaeJbpu' - b'qg0WtvHw31UoRZOnN_RZQYBwbAqteP0BApbos85syKE_VgfuNTtVRYkAlw5fwb_4Z' - b'WN-V-FFO_MrSGt71luX0rt-9ehNZFPHV1EuPc1YDQvZJ1XqPumewN-JAB5AABAA-a' - b'-CADBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP0BC1HB3w11jDft8De' - b'EhpuRVWZknabe5omZZA0s4QqX2o1O3gSM-PrjYr6lXlvnQ4TU_PaeF70kSo--LA22' - b'kL2csA-JAB4AADA-a-personal-CADBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQ' - b'YBwbAqteP0BAPZ92lw8F2_Ap81vFpyQsuTU9l7tOLI2ZmqanKOMVd2ar-m16JjA38' - b'PPH_mBFasyadIQgyun410RpxCUvsIBAH') + assert sig1 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--CADBE' + b'ByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSaRK4p0BCCKuUzhtasI04dqYQY4x' + b'7AqHMLP7Cbn2nqKFrR34_4zjJvT3HARSLhL_iUOSyoHMVv2-9mgtWuuW0oF4XS3n' + b'0C-JAB5AABAA-a-CADBEByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSaRK4p0B' + b'A0YOvEHllXmhlmv7ec_MShdDf2eZ3OvMDKE_zhfQsrFvU2ip1g-z04HZ097hN66r' + b'Lo5nqglvdCkxXJcYBqe8kD-JAB4AADA-a-personal-CADBEByXTm1zJZ1oIuvna' + b'd6hppC3iBrUmQLb0a7iXSaRK4p0BBjkD40acCOcdFuAM9d5FAgnfXqkZyI9haUTe' + b'VDO9vhPqgnRIXAUcTMxkY3eAfhX0ggy7n99sosla1nyVyBE_cK') # Sign with transferable identifier defaults to single signature on entire SAD sig2 = signing.ratify(sidHab, cred) - assert sig2 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSK' - b'lDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri' - b'":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLB' - b'SMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg3' - b'9j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFR' - b'D9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7x' - b'MwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36' - b'","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-",' - b'"n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalNam' - b'e":"Anna Jones","engagementContextRole":"Project Manager"}},"e":[' - b'{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC' - b'0N1jaQ42aKU5sHYTGFD"}]}-JAB6AABAAA--FABEKC8085pwSwzLwUGzh-HrEoFDw' - b'ZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh-HrEo' - b'FDwZnCJq27bVp5atdMT9o-AABAABf42ZzPJkYp6fm6yMhqYqSOg9IAf8dmxpBRq22' - b'GNZTQaalCJIuAxCVT2GRiqs7z7Z5kIPy5yhR3DYVx2Y_Rj8N') + assert sig2 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-JAB6AABAAA--FABEDT4jW7VYWenu1' + b'MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYW' + b'enu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAAA5PVjmjZFG-1meySYd8YrYhV' + b'XubpZd9DXRDJ1uIeFxjKiy_XbJAG7Yl00KxVAcObWDmBStuWLn-qoOI1YcFYoB') # Sign with transferable identifier with specific set of paths sig3 = signing.ratify(sidHab, cred, paths=paths) - assert sig3 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtSK' - b'lDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","ri' - b'":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1rLB' - b'SMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQzDXg3' - b'9j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFR' - b'D9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"EBmRy7x' - b'MwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84GM83MG36' - b'","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4GswcU6ov8-",' - b'"n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","personLegalNam' - b'e":"Anna Jones","engagementContextRole":"Project Manager"}},"e":[' - b'{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_ozM1uAnFvSfC' - b'0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--FABEKC8085pwSwzLw' - b'UGzh-HrEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSw' - b'zLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o-AABAABf42ZzPJkYp6fm6yMhqYqSOg9I' - b'Af8dmxpBRq22GNZTQaalCJIuAxCVT2GRiqs7z7Z5kIPy5yhR3DYVx2Y_Rj8N-JAB5' - b'AABAA-a-FABEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAA' - b'AAAAAAAAAAAAAAEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o-AABAAB' - b'B5IVZOhEfcH4TBQgOCyMgyQrJujtBBjT8K_zTPk0-FLMtTZuBgXV7jnLw6fDe6FWt' - b'zshh2HGCL_H_j4i1b9kF-JAB4AADA-a-personal-FABEKC8085pwSwzLwUGzh-Hr' - b'EoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh' - b'-HrEoFDwZnCJq27bVp5atdMT9o-AABAACUs3V7O3fRGmi_-X8xo8UFw7rGEQ0CYyf' - b'K7U1URYBwtW546a1pVJZ_c1wzcTXWLDj7OPPNrAQx2em5sNrOMPcH') + assert sig3 == (b'{"v":"ACDC10JSON0002e2_","d":"EIEbtplUZZrV3-jVAnOUcH-xcxOCuiJqtS' + b'KlDJSxUJLW","i":"EBNHFK056fqNSG_MDE7d_Eqk0bazefvd4eeQLMPPNBnM","' + b'ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EAllThM1' + b'rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EHpYRwyyu23jcGQz' + b'DXg39j1b7U4_VB8sW5_YsqD6YNpY","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0J' + b'p6ZFRD9dTb4-5c0","dt":"2021-06-27T21:26:21.233257+00:00","ri":"E' + b'BmRy7xMwsxUelUauaXtMxTfPAMPAI6FkekwlOjkggt","LEI":"254900OPPU84G' + b'M83MG36","personal":{"d":"EMUsc87LxpYTultDtf_pOQKYvDYA_HQxy4Gswc' + b'U6ov8-","n":"Q8rNaKITBLLA96Euh5M5v4o3fRl1Bc54xdM-bOIHUjY","perso' + b'nLegalName":"Anna Jones","engagementContextRole":"Project Manage' + b'r"}},"e":[{"qualifiedvLEIIssuervLEICredential":"EAtyThM1rLBSMZ_o' + b'zM1uAnFvSfC0N1jaQ42aKU5sHYTGFD"}]}-KAD6AABAAA--JAB6AABAAA--FABED' + b'T4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAA' + b'AAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAAA5PVjmjZFG-1' + b'meySYd8YrYhVXubpZd9DXRDJ1uIeFxjKiy_XbJAG7Yl00KxVAcObWDmBStuWLn-q' + b'oOI1YcFYoB-JAB5AABAA-a-FABEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUd' + b'xKLWax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw' + b'0RUdxKLWax-AABAABYXrbXx0VXn_qs1AVUzVqJ19lfnN1aBoxaLjlcgHaCgCOaY_' + b'AUPOuEyE9EFgA_VLCHnPi7TdZhbXyG09Due24P-JAB4AADA-a-personal-FABED' + b'T4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAA' + b'AAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAACNv9Bg6Gg9oi' + b'pvuArddJgY1uNfp7uFO8ifkwKHhQargXScvQuvYIV7-VZM4Xu0vNOWaKA-mDmfei' + b'sRYanh_gEJ') # Test multisig identifier with configing.openCF(name="mel", base="mel", temp=True) as cf, \ - habbing.openHby(name="mel", temp=True, salt=coring.Salter(raw=b'0123456789abcdef').qb64b, + habbing.openHby(name="mel", temp=True, salt=core.Salter(raw=b'0123456789abcdef').qb64b, base="mel", cf=cf) as hby: hab = hby.makeHab(name="mel", icount=3, isith='3', ncount=3, nsith='3') seeder.seedSchema(hby.db) @@ -154,18 +157,19 @@ def test_sad_signature(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): # Sign with multisig transferable identifier defaults to single signature on entire SAD sig1 = signing.ratify(hab=hab, serder=cred) - assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EFnCQiKLTr37bhuFrvbE88GhWyQ-LUryYNiDjO4' - b'cW1Nt","i":"EKMHlh4epBApuYP-3-A_ZldeImCa6WxLe8Nmzhy-SvWB","ri":"ELIc0' - b'Va2OSemOpuiD2Fxfzd2yXg6CWibjOCJY3vNFLb9","s":"EMQWEcCnVRk1hatTNyK3sIy' - b'kYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EEBT5231zltSwAzucatIlavzJyE43F3op2l6' - b'aB6or1FP","dt":"2021-06-09T17:35:54.169967+00:00","i":"EAwC_-ISX9helB' - b'DUSKU5j_VEU3G0Jp6ZFRD9dTb4-5c0","LEI":"254900OPPU84GM83MG36"},"e":{}}' - b'-JAB6AABAAA--FABEKMHlh4epBApuYP-3-A_ZldeImCa6WxLe8Nmzhy-SvWB0AAAAAAAA' - b'AAAAAAAAAAAAAAAEKMHlh4epBApuYP-3-A_ZldeImCa6WxLe8Nmzhy-SvWB-AADAADV_a' - b'TlO9rV3bxKCiSr4zqqu0Jg1Q3u6Fk2pWxrctkXepRRlDCfKeKmlMQHnNJ8r6vRQp8Eslv' - b'h33uSBdS_MlIPABC--losKLa_YfcwGBIabsq8g5VABrsmnXkZMT7eB-jqK6bdB4Zs_863' - b'la0mm_DeMwS1KXomLb_j1zCmgZ3RJPAPACA539yer3U8JQlcgXrdbPlR-1kADcFA4bsN_' - b'klRSu7p61y-Z2CS5d7Aitrc7yq00YIG_u-v7OToChDC3TsVCR4D') + assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EPA-T12T4d8sHaj2IyvK2ReIs5KREYN24W' + b'tJbYL-d2HQ","i":"ELqU3H49LwniPBMVMGqWjSfh_8mFRntbXVlBQirijKMi","' + b'ri":"ENw4eVQH4l7uHSYa50yWh1AW8Wj22V77KHL7Wg55F5gB","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EEBT5231zltSwAzu' + b'catIlavzJyE43F3op2l6aB6or1FP","dt":"2021-06-09T17:35:54.169967+0' + b'0:00","i":"EAwC_-ISX9helBDUSKU5j_VEU3G0Jp6ZFRD9dTb4-5c0","LEI":"' + b'254900OPPU84GM83MG36"},"e":{}}-JAB6AABAAA--FABELqU3H49LwniPBMVMG' + b'qWjSfh_8mFRntbXVlBQirijKMi0AAAAAAAAAAAAAAAAAAAAAAAELqU3H49LwniPB' + b'MVMGqWjSfh_8mFRntbXVlBQirijKMi-AADAAB7s8EP1DushDLnCeo3xcud-y_fXm' + b'X4AL93H8hUAFDv4o2x-5_GHmpUWC9lt86DW_sUX1rJA1OsPGaCYXiFFOoHABD3bW' + b'HcN4gL4tgtN9kl8737NzW18wIIE2GEEvKh2Z45Ai5sIynHzU0q6mCD3LTIRb5DfA' + b'aazZfDFlqgLYtyBVIPACDpAkl7FGW7pQ5ZLW6rCLTBYUfM_bagl7Fwo_GJhYNAiR' + b'3D52Y6QFhqQDn-F_FJZSAlQUZZGmT0ZJeIEHg5qwwH') """End Test""" @@ -176,8 +180,8 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs seed1 = (b'\x83B~\x04\x94\xe3\xceUQy\x11f\x0c\x93]\x1e\xbf\xacQ\xb5\xd6Y^\xa2E\xfa\x015' b'\x98Y\xdd\xe8') - signer0 = coring.Signer(raw=seed, transferable=True) # original signing keypair non transferable - signer1 = coring.Signer(raw=seed1) # next signing keypair transferable is default + signer0 = core.Signer(raw=seed, transferable=True) # original signing keypair non transferable + signer1 = core.Signer(raw=seed1) # next signing keypair transferable is default keys0 = [signer0.verfer.qb64] nxt1 = [coring.Diger(ser=signer1.verfer.qb64b).qb64] # dfault sith is 1 serder = eventing.incept(keys=keys0, ndigs=nxt1, code=coring.MtrDex.Blake3_256, intive=True) @@ -208,20 +212,21 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs # Sign with non-transferable identifier, defaults to single signature on entire SAD sig0 = bytearray(cred.raw) - sig0.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + sig0.extend(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) sig0.extend(coring.Prefixer(qb64=issuer.regk).qb64b) sig0.extend(seqner.qb64b) sig0.extend(coring.Saider(qb64=issuer.regd).qb64b) - assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EK88fyN65bfA63o1jgeOGKeIxw6sTJEwwU' - b'3ycpjdtCUD","i":"EKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o","' - b'ri":"ENzh5cyGjFhQYuIXuheXV2wkKp23rkxYI7wbEBQIyqhP","s":"EMQWEcCn' + assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EEs1BzDdw0h5VMhg5KUo904UyCdaMT1ut_' + b'_vN5G879pP","i":"EDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax","' + b'ri":"ECBNRQkp71gi2mTqZu2cp-xe6CrWlvJ5pRoBIBnRQbGp","s":"EMQWEcCn' b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcu' b'vv8j5F4FWRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+0' b'0:00","i":"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"' - b'254900OPPU84GM83MG36"},"e":{}}-IABENzh5cyGjFhQYuIXuheXV2wkKp23rk' - b'xYI7wbEBQIyqhP0AAAAAAAAAAAAAAAAAAAAAABENzh5cyGjFhQYuIXuheXV2wkKp' - b'23rkxYI7wbEBQIyqhP') + b'254900OPPU84GM83MG36"},"e":{}}-IABECBNRQkp71gi2mTqZu2cp-xe6CrWlv' + b'J5pRoBIBnRQbGp0AAAAAAAAAAAAAAAAAAAAAABECBNRQkp71gi2mTqZu2cp-xe6C' + b'rWlvJ5pRoBIBnRQbGp') iss = issuer.issue(said=cred.said) rseal = SealEvent(iss.pre, "0", iss.said)._asdict() @@ -261,23 +266,23 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs # Bad magic values here but can't figure out where looks like Sadder Said seeder # is using a bad magic value sig1 = signing.ratify(hab=hab, serder=cred, paths=[[], ["a"], ["a", "i"]]) - assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EK88fyN65bfA63o1jgeOGKeIxw6sTJEwwU3y' - b'cpjdtCUD","i":"EKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o","ri":' - b'"ENzh5cyGjFhQYuIXuheXV2wkKp23rkxYI7wbEBQIyqhP","s":"EMQWEcCnVRk1ha' - b'tTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcuvv8j5F4F' - b'WRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+00:00","i":' - b'"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"254900OPPU84' - b'GM83MG36"},"e":{}}-KAD6AABAAA--JAB6AABAAA--FABEKC8085pwSwzLwUGzh-H' - b'rEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh' - b'-HrEoFDwZnCJq27bVp5atdMT9o-AABAACIRDrYzCyMB5jBHY9jwfT4KEb7kx_vYgHJ' - b'7LDsiQRD-Roj5bGfJXj6PAo5TS36t4kWmiBhpvqLgb2l9vUhpiUK-JAB5AABAA-a-F' - b'ABEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o0AAAAAAAAAAAAAAAAAAA' - b'AAAAEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o-AABAAB80PrmAUGj_i' - b'ATyLY-kzdpI6omm5X05EsdkRZGymwVn62-1nijoSh0dlUo6rGOoywUQWu-eZ0i5PuH' - b'skgV9nwP-JAB4AAB-a-i-FABEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT' - b'9o0AAAAAAAAAAAAAAAAAAAAAAAEKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atd' - b'MT9o-AABAABsIw-EgCMnex1m7Qm8RkU4jMGAV3wNGyD_CxfetmMp-iGBLhZ5wArAw6' - b'_Qdg75K_NMTKVV4hv7bWw3OvJnNY8A') + assert sig1 == (b'{"v":"ACDC10JSON00019e_","d":"EEs1BzDdw0h5VMhg5KUo904UyCdaMT1ut_' + b'_vN5G879pP","i":"EDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax","' + b'ri":"ECBNRQkp71gi2mTqZu2cp-xe6CrWlvJ5pRoBIBnRQbGp","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcu' + b'vv8j5F4FWRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+0' + b'0:00","i":"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"' + b'254900OPPU84GM83MG36"},"e":{}}-KAD6AABAAA--JAB6AABAAA--FABEDT4jW' + b'7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAAAAED' + b'T4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAAA0vjmyMr3yp2DPHG' + b'fWczKx3rTd0EQndtntndKMEucbuUBh9B-KOIifWpzfpvvdOl4seK_nkfLsYt1NXx' + b'EMLqIC-JAB5AABAA-a-FABEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUdxKLW' + b'ax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYWenu1MrgJJEZWB0ScqZdVmDQw0RUd' + b'xKLWax-AABAAAMSXJBtyWP60AH_SPThf2kW7J7VNudUEwTX6fvD-EhE-DypJogi-' + b'SAm-eJ7kyo5v1zbgeacUDo2lYK3BZ1POcE-JAB4AAB-a-i-FABEDT4jW7VYWenu1' + b'MrgJJEZWB0ScqZdVmDQw0RUdxKLWax0AAAAAAAAAAAAAAAAAAAAAAAEDT4jW7VYW' + b'enu1MrgJJEZWB0ScqZdVmDQw0RUdxKLWax-AABAADvLjrNYsoHiYyrSzaFIrPd88' + b'gqaTIWqwnsVK_HfmR27PJgLXv36zqT4lNDTHyTjFWcA7tBghCkG_7aBsDkW38P') # signing SAD with non-transferable identifier with habbing.openHab(name="wan", temp=True, salt=b'0123456789abcdef', transferable=False) as (hby, hab): @@ -289,15 +294,15 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs # Sign with non-transferable identifier sig0 = signing.ratify(hab=hab, serder=cred) - assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EBFPlqNJ4FovUoaGiDg3TWAlabvbUv2OnV' - b'wF8zPuaey7","i":"BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP","' + assert sig0 == (b'{"v":"ACDC10JSON00019e_","d":"EDvgu38Hmuprln_wc57VYkeJ9Dh0PlcOon' + b'qHhXjh6Yql","i":"BEByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSaRK4p","' b'ri":"Eg1H4eN7P5ndJAWtcymq3ZrYZwQsBRYd3-VuZ6wMAwxE","s":"EMQWEcCn' b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFyxk35e1r5G9pcu' b'vv8j5F4FWRHD8xlZ_E4rWPdlVASI","dt":"2021-06-09T17:35:54.169967+0' b'0:00","i":"EIflL4H4134zYoRM6ls6Q086RLC_BhfNFh5uk-WxvhsL","LEI":"' - b'254900OPPU84GM83MG36"},"e":{}}-JAB6AABAAA--CABBAbSj3jfaeJbpuqg0W' - b'tvHw31UoRZOnN_RZQYBwbAqteP0BD1mbdIQtZtdLvSdR7in7XOt-d2ALjyOTvVbQ' - b'6AMLcyd5H7m-rPM8RshHoREehbr5S82w7b844-ykD5yJHyqhIG') + b'254900OPPU84GM83MG36"},"e":{}}-JAB6AABAAA--CABBEByXTm1zJZ1oIuvna' + b'd6hppC3iBrUmQLb0a7iXSaRK4p0BBfvm3yAoWIysgYFm4nQ7zQY6_Obo50YHKRVZ' + b'0aqphSHLn4H0YuZK-0dQEsUjAOZuvPK6TkR8w1hWxOE-lXug8C') pather = coring.Pather(path=["a", "b", "c"]) cigars = hab.sign(ser=cred.raw, @@ -306,14 +311,14 @@ def test_signature_transposition(seeder, mockCoringRandomNonce, mockHelpingNowIs sadcigars = [(pather, cigars)] tp = eventing.proofize(sadcigars=sadcigars, pipelined=True) - assert tp == (b'-VAm-JAB5AACAA-a-b-c-CABBAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP' - b'0BD1mbdIQtZtdLvSdR7in7XOt-d2ALjyOTvVbQ6AMLcyd5H7m-rPM8RshHoREehbr5S' - b'82w7b844-ykD5yJHyqhIG') + assert tp == (b'-VAm-JAB5AACAA-a-b-c-CABBEByXTm1zJZ1oIuvnad6hppC3iBrUmQLb0a7iXSa' + b'RK4p0BBfvm3yAoWIysgYFm4nQ7zQY6_Obo50YHKRVZ0aqphSHLn4H0YuZK-0dQEs' + b'UjAOZuvPK6TkR8w1hWxOE-lXug8C') """End Test""" def test_signatory(): - salt = coring.Salter(raw=b'0123456789abcdef') # init sig Salter + salt = core.Salter(raw=b'0123456789abcdef') # init sig Salter with basing.openDB(name="sig") as db, keeping.openKS(name="sig") as ks, \ habbing.openHab(name="sig", salt=salt.raw) as (sigHby, sigHab): diff --git a/tests/app/test_storing.py b/tests/app/test_storing.py index 82bf39352..b0208ed19 100644 --- a/tests/app/test_storing.py +++ b/tests/app/test_storing.py @@ -1,17 +1,20 @@ # -*- encoding: utf-8 -*- """ -tests.peer.mailboxing +tests.app.storing """ import os import lmdb +import pytest from keri.app import keeping from keri.core import coring, serdering -from keri.db import dbing, basing +from keri.db import dbing, basing, subing +from keri.db.basing import KERIBaserMapSizeKey +from keri.db.dbing import LMDBer, KERILMDBMapSizeKey from keri.peer import exchanging -from keri.app.storing import Mailboxer +from keri.app.storing import Mailboxer, KERIMailboxerMapSizeKey def test_mailboxing(): @@ -28,7 +31,8 @@ def test_mailboxing(): assert mber.env.path() == mber.path assert os.path.exists(mber.path) - assert isinstance(mber.tpcs, lmdb._Database) + #assert isinstance(mber.tpcs, lmdb._Database) + assert isinstance(mber.tpcs, subing.OnSuber) mber.close(clear=True) assert not os.path.exists(mber.path) @@ -85,7 +89,7 @@ def test_mailboxing(): mber.storeMsg(topic=dest.qb64b, msg=exn.raw) msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=0): + for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b): msgs.append((fn, msg)) assert(len(msgs)) == 10 @@ -95,20 +99,56 @@ def test_mailboxing(): d = exn.ked["a"] assert d["b"] == idx - msgs = [] - for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): - msgs.append(msg) + #msgs = [] + #for fn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=10): + #msgs.append(msg) - assert(len(msgs)) == 0 + #assert(len(msgs)) == 0 - msgs = [] - for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): - msgs.append((tn, msg)) + #msgs = [] + #for tn, topic, msg in mber.cloneTopicIter(topic=dest.qb64b, fn=4): + #msgs.append((tn, msg)) + + #assert(len(msgs)) == 6 + #assert msgs[0][0] == 4 - assert(len(msgs)) == 6 - assert msgs[0][0] == 4 +def test_mailbox_db_size_set_from_env_var(): + # Clear environment before test + if KERILMDBMapSizeKey in os.environ: + os.environ.pop(KERILMDBMapSizeKey) + if KERIMailboxerMapSizeKey in os.environ: + os.environ.pop(KERIMailboxerMapSizeKey) + + new_map_size = 10737418240 + # Default map size works + mber = Mailboxer() + assert mber.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert mber.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + mber.close() + # Specific map size works + os.environ[KERIMailboxerMapSizeKey] = f"{new_map_size}" + mber = Mailboxer() + assert mber.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERIMailboxerMapSizeKey) + mber.close() + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERILMDBMapSizeKey] = f"{baser_map_size}" + + mber = Mailboxer() + assert mber.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + mber.close() + + # Bad map size throws + os.environ[KERIMailboxerMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + Mailboxer() + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERILMDBMapSizeKey) + os.environ.pop(KERIMailboxerMapSizeKey) if __name__ == '__main__': test_mailboxing() diff --git a/tests/app/test_tocking.py b/tests/app/test_tocking.py new file mode 100644 index 000000000..1a03bcae8 --- /dev/null +++ b/tests/app/test_tocking.py @@ -0,0 +1,258 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.test_tocking module + +Tests for centralized tock configuration. +""" +import os +import importlib + +import pytest + + +def test_tocking_defaults(): + """ + Test default tock values when no environment variables are set + """ + # Clear any existing env vars that might affect the test + env_keys = [ + "KERI_WITNESS_MSG_TOCK", + "KERI_WITNESS_ESCROW_TOCK", + "KERI_WITNESS_CUE_TOCK", + "KERI_INDIRECTOR_MSG_TOCK", + "KERI_INDIRECTOR_CUE_TOCK", + "KERI_INDIRECTOR_ESCROW_TOCK", + "KERI_MAILBOX_POLL_TOCK", + "KERI_MAILBOX_MSG_TOCK", + "KERI_MAILBOX_ESCROW_TOCK", + "KERI_POLLER_EVENT_TOCK", + "KERI_RECEIPT_INTERCEPT_TOCK", + "KERI_REACTOR_MSG_TOCK", + "KERI_REACTOR_CUE_TOCK", + "KERI_REACTOR_ESCROW_TOCK", + "KERI_REACTANT_MSG_TOCK", + "KERI_REACTANT_CUE_TOCK", + "KERI_REACTANT_ESCROW_TOCK", + "KERI_RESPONDANT_CUE_TOCK", + "KERI_WITNESS_RECEIPTOR_TOCK", + "KERI_WITNESS_INQUISITOR_TOCK", + "KERI_WITNESS_PUBLISHER_TOCK", + ] + + # Save and clear existing env vars + saved_env = {} + for key in env_keys: + if key in os.environ: + saved_env[key] = os.environ.pop(key) + + try: + # Reload module to pick up cleared env vars + from keri.app import tocking + importlib.reload(tocking) + + # Witness component tocks + assert tocking.WitnessMsgTock == 0.0 + assert tocking.WitnessEscrowTock == 1.0 + assert tocking.WitnessCueTock == 0.25 + + # Indirector component tocks + assert tocking.IndirectorMsgTock == 0.0 + assert tocking.IndirectorCueTock == 0.25 + assert tocking.IndirectorEscrowTock == 1.0 + + # MailboxDirector component tocks + assert tocking.MailboxPollTock == 0.5 + assert tocking.MailboxMsgTock == 0.0 + assert tocking.MailboxEscrowTock == 1.0 + + # Poller component tocks + assert tocking.PollerEventTock == 0.5 + + # ReceiptEnd component tocks + assert tocking.ReceiptInterceptTock == 0.1 + + # Reactor component tocks + assert tocking.ReactorMsgTock == 0.0 + assert tocking.ReactorCueTock == 0.25 + assert tocking.ReactorEscrowTock == 1.0 + + # Reactant component tocks + assert tocking.ReactantMsgTock == 0.0 + assert tocking.ReactantCueTock == 0.25 + assert tocking.ReactantEscrowTock == 1.0 + + # Respondant component tocks + assert tocking.RespondantCueTock == 0.25 + + # WitnessReceiptor component tocks + assert tocking.WitnessReceiptorTock == 0.0 + + # WitnessInquisitor component tocks + assert tocking.WitnessInquisitorTock == 1.0 + + # WitnessPublisher component tocks + assert tocking.WitnessPublisherTock == 0.25 + + finally: + # Restore saved env vars + for key, value in saved_env.items(): + os.environ[key] = value + # Reload to restore original state + importlib.reload(tocking) + + """Done Test""" + + +def test_tocking_env_override(): + """ + Test that environment variables properly override default tock values + """ + # Save existing env vars + env_overrides = { + "KERI_WITNESS_MSG_TOCK": "2.5", + "KERI_WITNESS_ESCROW_TOCK": "3.0", + "KERI_WITNESS_CUE_TOCK": "0.5", + "KERI_INDIRECTOR_MSG_TOCK": "1.0", + "KERI_INDIRECTOR_CUE_TOCK": "0.75", + "KERI_INDIRECTOR_ESCROW_TOCK": "2.0", + "KERI_MAILBOX_POLL_TOCK": "1.5", + "KERI_MAILBOX_MSG_TOCK": "0.1", + "KERI_MAILBOX_ESCROW_TOCK": "2.5", + "KERI_POLLER_EVENT_TOCK": "1.0", + "KERI_RECEIPT_INTERCEPT_TOCK": "0.5", + "KERI_REACTOR_MSG_TOCK": "0.5", + "KERI_REACTOR_CUE_TOCK": "0.5", + "KERI_REACTOR_ESCROW_TOCK": "2.0", + "KERI_REACTANT_MSG_TOCK": "0.5", + "KERI_REACTANT_CUE_TOCK": "0.5", + "KERI_REACTANT_ESCROW_TOCK": "2.0", + "KERI_RESPONDANT_CUE_TOCK": "0.5", + "KERI_WITNESS_RECEIPTOR_TOCK": "0.5", + "KERI_WITNESS_INQUISITOR_TOCK": "2.0", + "KERI_WITNESS_PUBLISHER_TOCK": "0.5", + } + + # Save and set env vars + saved_env = {} + for key, value in env_overrides.items(): + if key in os.environ: + saved_env[key] = os.environ[key] + os.environ[key] = value + + try: + # Reload module to pick up new env vars + from keri.app import tocking + importlib.reload(tocking) + + # Witness component tocks + assert tocking.WitnessMsgTock == 2.5 + assert tocking.WitnessEscrowTock == 3.0 + assert tocking.WitnessCueTock == 0.5 + + # Indirector component tocks + assert tocking.IndirectorMsgTock == 1.0 + assert tocking.IndirectorCueTock == 0.75 + assert tocking.IndirectorEscrowTock == 2.0 + + # MailboxDirector component tocks + assert tocking.MailboxPollTock == 1.5 + assert tocking.MailboxMsgTock == 0.1 + assert tocking.MailboxEscrowTock == 2.5 + + # Poller component tocks + assert tocking.PollerEventTock == 1.0 + + # ReceiptEnd component tocks + assert tocking.ReceiptInterceptTock == 0.5 + + # Reactor component tocks + assert tocking.ReactorMsgTock == 0.5 + assert tocking.ReactorCueTock == 0.5 + assert tocking.ReactorEscrowTock == 2.0 + + # Reactant component tocks + assert tocking.ReactantMsgTock == 0.5 + assert tocking.ReactantCueTock == 0.5 + assert tocking.ReactantEscrowTock == 2.0 + + # Respondant component tocks + assert tocking.RespondantCueTock == 0.5 + + # WitnessReceiptor component tocks + assert tocking.WitnessReceiptorTock == 0.5 + + # WitnessInquisitor component tocks + assert tocking.WitnessInquisitorTock == 2.0 + + # WitnessPublisher component tocks + assert tocking.WitnessPublisherTock == 0.5 + + finally: + # Restore or remove env vars + for key in env_overrides: + if key in saved_env: + os.environ[key] = saved_env[key] + else: + os.environ.pop(key, None) + # Reload to restore original state + importlib.reload(tocking) + + """Done Test""" + + +def test_tocking_keys(): + """ + Test that environment variable keys are correctly defined + """ + from keri.app import tocking + + # Witness component keys + assert tocking.KERI_WITNESS_MSG_TOCK_KEY == "KERI_WITNESS_MSG_TOCK" + assert tocking.KERI_WITNESS_ESCROW_TOCK_KEY == "KERI_WITNESS_ESCROW_TOCK" + assert tocking.KERI_WITNESS_CUE_TOCK_KEY == "KERI_WITNESS_CUE_TOCK" + + # Indirector component keys + assert tocking.KERI_INDIRECTOR_MSG_TOCK_KEY == "KERI_INDIRECTOR_MSG_TOCK" + assert tocking.KERI_INDIRECTOR_CUE_TOCK_KEY == "KERI_INDIRECTOR_CUE_TOCK" + assert tocking.KERI_INDIRECTOR_ESCROW_TOCK_KEY == "KERI_INDIRECTOR_ESCROW_TOCK" + + # MailboxDirector component keys + assert tocking.KERI_MAILBOX_POLL_TOCK_KEY == "KERI_MAILBOX_POLL_TOCK" + assert tocking.KERI_MAILBOX_MSG_TOCK_KEY == "KERI_MAILBOX_MSG_TOCK" + assert tocking.KERI_MAILBOX_ESCROW_TOCK_KEY == "KERI_MAILBOX_ESCROW_TOCK" + + # Poller component keys + assert tocking.KERI_POLLER_EVENT_TOCK_KEY == "KERI_POLLER_EVENT_TOCK" + + # ReceiptEnd component keys + assert tocking.KERI_RECEIPT_INTERCEPT_TOCK_KEY == "KERI_RECEIPT_INTERCEPT_TOCK" + + # Reactor component keys + assert tocking.KERI_REACTOR_MSG_TOCK_KEY == "KERI_REACTOR_MSG_TOCK" + assert tocking.KERI_REACTOR_CUE_TOCK_KEY == "KERI_REACTOR_CUE_TOCK" + assert tocking.KERI_REACTOR_ESCROW_TOCK_KEY == "KERI_REACTOR_ESCROW_TOCK" + + # Reactant component keys + assert tocking.KERI_REACTANT_MSG_TOCK_KEY == "KERI_REACTANT_MSG_TOCK" + assert tocking.KERI_REACTANT_CUE_TOCK_KEY == "KERI_REACTANT_CUE_TOCK" + assert tocking.KERI_REACTANT_ESCROW_TOCK_KEY == "KERI_REACTANT_ESCROW_TOCK" + + # Respondant component keys + assert tocking.KERI_RESPONDANT_CUE_TOCK_KEY == "KERI_RESPONDANT_CUE_TOCK" + + # WitnessReceiptor component keys + assert tocking.KERI_WITNESS_RECEIPTOR_TOCK_KEY == "KERI_WITNESS_RECEIPTOR_TOCK" + + # WitnessInquisitor component keys + assert tocking.KERI_WITNESS_INQUISITOR_TOCK_KEY == "KERI_WITNESS_INQUISITOR_TOCK" + + # WitnessPublisher component keys + assert tocking.KERI_WITNESS_PUBLISHER_TOCK_KEY == "KERI_WITNESS_PUBLISHER_TOCK" + + """Done Test""" + + +if __name__ == "__main__": + test_tocking_defaults() + test_tocking_env_override() + test_tocking_keys() diff --git a/tests/app/test_watching.py b/tests/app/test_watching.py new file mode 100644 index 000000000..de6ca0da9 --- /dev/null +++ b/tests/app/test_watching.py @@ -0,0 +1,167 @@ +# -*- encoding: utf-8 -*- +""" +tests.app.watching + +""" +from dataclasses import asdict + +import pytest + +from keri import core +from keri.app import watching, habbing +from keri.app.watching import DiffState +from keri.core import coring +from keri.db.basing import KeyStateRecord, ObservedRecord + + +def test_diffstate(): + d0 = {'vn': [1, 0], + 'i': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 's': '0', + 'p': 'ElsHFkbZQjRb7xHnuE-wyiarIZ9j-1CEQ89I0E3WevcE', + 'd': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'f': '0', + 'dt': '2021-06-09T17:35:54.169967+00:00', + 'et': '2021-06-09T17:35:54.169967+00:00', + 'kt': '1', + 'k': ["D-HwiqmaETxls3vAVSh0xpXYTs94NUJX6juupWj_EgsA"], + 'nt': '1', + 'n': ["ED6lKZwg-BWl_jlCrjosQkOEhqKD4BJnlqYqWmhqPhaU"], + 'bt': '0', + 'b': [], + 'c': [], + 'ee': { + 's': '0', + 'd': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'br': [], + 'ba': [] + }, + 'di': ''} + + ksr0 = KeyStateRecord(**d0) + d1 = {'vn': [1, 0], + 'i': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 's': '0', + 'p': 'ElsHFkbZQjRb7xHnuE-wyiarIZ9j-1CEQ89I0E3WevcE', + 'd': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'f': '0', + 'dt': '2021-06-09T17:35:54.169967+00:00', + 'et': '2021-06-09T17:35:54.169967+00:00', + 'kt': '1', + 'k': ["DxVTxls3vAwiqmaEXYTs94NUJX6juVSh0xpupEgsAWj_"], + 'nt': '1', + 'n': ["ED6lKZwg-BWl_jlCrjosQkOEhqKD4BJnlqYqWmhqPhaU"], + 'bt': '0', + 'b': [], + 'c': [], + 'ee': { + 's': '0', + 'd': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'br': [], + 'ba': [] + }, + 'di': ''} + ksr1 = KeyStateRecord(**d1) + + wat = "BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s" + diffstate = watching.diffState(wat, ksr0, ksr1) + + # Sequence numbers are the same, digest different == duplicitous + assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 'sn': 0, + 'state': 'duplicitous', + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} + + # Same state == event + diffstate = watching.diffState(wat, ksr0, ksr0) + assert asdict(diffstate) == {'dig': 'EBiIFxr_o1b4x1YR21PblAFpFG61qDghqFBDyVSOXYW0', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 'sn': 0, + 'state': 'even', + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} + + ksr1.s = "2" + diffstate = watching.diffState(wat, ksr0, ksr1) + + # Sequence numbers are the same, digest different == duplicitous + assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 'sn': 2, + 'state': 'ahead', + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} + + ksr0.s = "3" + diffstate = watching.diffState(wat, ksr0, ksr1) + + # Sequence numbers are the same, digest different == duplicitous + assert asdict(diffstate) == {'dig': 'Ey2pXEnaoQVwxA4jB6k0QH5G2Us-0juFL5hOAHAwIEkc', + 'pre': 'EZ-i0d8JZAoTNZH3ULaU6JR2nmwyvYAfSVPzhzS6b5CM', + 'sn': 2, + 'state': 'behind', + 'wit': 'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'} + + +def test_adjudicator(): + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + with habbing.openHby(name="test", base="test", salt=default_salt) as hby: + hab = hby.makeHab("test") + assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + wat = "BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s" + saider = coring.Saider(qb64b=b'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ') + + ksr = hab.kever.state() + ksr0 = KeyStateRecord(**asdict(ksr)) + + hab.db.knas.pin(keys=(hab.pre, wat), val=saider) + hab.db.ksns.pin(keys=(saider.qb64, ), val=ksr0) + hab.db.obvs.pin(keys=(hab.pre, wat, hab.pre), val=ObservedRecord(enabled=True)) + + adj = watching.Adjudicator(hby=hby, hab=hab) + + adj.adjudicate(hab.pre, 1) + assert len(adj.cues) == 1 + cue = adj.cues.pull() + + assert cue == {'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'kin': 'keyStateConsistent', + 'oid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'states': [DiffState(pre="EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3", + wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + state='even', + sn=0, + dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], + 'wids': {'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'}} + + hab.rotate() + + adj.adjudicate(hab.pre, 1) + assert len(adj.cues) == 1 + cue = adj.cues.pull() + assert cue == {'behind': [DiffState(pre="EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3", + wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + state='behind', + sn=0, + dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], + 'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'kin': 'keyStateLagging', + 'oid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'wids': {'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'}} + + ksr0.s = '1' + hab.db.ksns.pin(keys=(saider.qb64, ), val=ksr0) + adj.adjudicate(hab.pre, 1) + assert len(adj.cues) == 1 + cue = adj.cues.pull() + assert cue == {'cid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'dups': [DiffState(pre="EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3", + wit='BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s', + state='duplicitous', + sn=1, + dig='EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3')], + 'kin': 'keyStateDuplicitous', + 'oid': 'EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3', + 'wids': {'BbIg_3-11d3PYxSInLN-Q9_T2axD6kkXd3XRgbGZTm6s'}} + + with pytest.raises(ValueError): + adj.adjudicate(hab.pre, 2) diff --git a/tests/comply/test_direct_mode.py b/tests/comply/test_direct_mode.py index 6ca1c4823..3bb8a69ef 100644 --- a/tests/comply/test_direct_mode.py +++ b/tests/comply/test_direct_mode.py @@ -2,11 +2,15 @@ import os -from keri.core.coring import Salter, MtrDex -from keri.core.coring import Seqner -from keri.app.keeping import Manager, openKS +from keri import core from keri.core import eventing, parsing +from keri.core import MtrDex +from keri.core.coring import Seqner + from keri.core.eventing import (incept, rotate, interact, messagize, SealEvent, receipt) + +from keri.app.keeping import Manager, openKS + from keri.db.dbing import dgKey, snKey from keri.db.basing import openDB @@ -24,10 +28,10 @@ def test_direct_mode_with_manager(): # but goes both ways once initiated. # set of secrets (seeds for private keys) - coeSalt = Salter(raw=b'0123456789abcdea').qb64 + coeSalt = core.Salter(raw=b'0123456789abcdea').qb64 # set of secrets (seeds for private keys) - valSalt = Salter(raw=b'1123456789abcdea').qb64 + valSalt = core.Salter(raw=b'1123456789abcdea').qb64 with openDB(name="controller") as coeLogger, openDB(name="validator") as valLogger, \ diff --git a/tests/conftest.py b/tests/conftest.py index 89dbaab37..330394f07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ from hio.base import doing from keri import kering -from keri.core import scheming, coring, routing, eventing, parsing +from keri.core import scheming, coring, routing, eventing, parsing, signing from keri.db import basing from keri.help import helping from keri import help @@ -68,7 +68,12 @@ def mockCoringRandomNonce(monkeypatch): def mockRandomNonce(): return "A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U" + @property + def mockRandomSalt(self): + return "0AAUiJMii_rPXXCiLTEEaDT7" + monkeypatch.setattr(coring, "randomNonce", mockRandomNonce) + monkeypatch.setattr(signing.Salter, "qb64", mockRandomSalt) @pytest.fixture @@ -357,7 +362,7 @@ def __init__(self, command, **kwa): self.command = command super(CommandDoer, self).__init__(doers=[doing.doify(self.cmdDo)], **kwa) - def cmdDo(self, tymth, tock=0.0): + def cmdDo(self, tymth, tock=0.0, **kwa): """ Execute single command from .command by parsing and executing the resulting doers """ # enter context diff --git a/tests/core/test_bare.py b/tests/core/test_bare.py index 5794d22ce..c6b161e73 100644 --- a/tests/core/test_bare.py +++ b/tests/core/test_bare.py @@ -8,12 +8,16 @@ """ from keri import kering +from keri import help + +from keri import core from keri.core import eventing -from keri.core.coring import MtrDex, Salter, Diger + +from keri.core.coring import MtrDex, Diger from keri.core.eventing import (SealEvent, messagize) -from keri import help + logger = help.ogler.getLogger() @@ -45,7 +49,7 @@ def test_bare(): # use same salter for all but different path # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create transferable key pair for controller of service endpoint designation signerC = salter.signer(path="C", temp=True) diff --git a/tests/core/test_coring.py b/tests/core/test_coring.py index 7ec33df4a..e41189493 100644 --- a/tests/core/test_coring.py +++ b/tests/core/test_coring.py @@ -3,7 +3,7 @@ tests.core.test_coring module """ -import dataclasses +from dataclasses import dataclass, asdict, astuple import hashlib import json from base64 import urlsafe_b64decode as decodeB64 @@ -23,330 +23,194 @@ from cryptography.hazmat.primitives import hashes from cryptography import exceptions -from keri.core import coring -from keri.core import eventing -from keri.core.coring import (Ilkage, Ilks, Labels, Saids, Protos, Protocolage, - Sadder, Tholder, Seqner, - NumDex, Number, Siger, Dater, Bexter) -from keri.core.coring import Serialage, Serials, Tiers, Vstrings -from keri.core.coring import (Sizage, MtrDex, Matter, Xizage, IdrDex, IdxSigDex, - IdxCrtSigDex, IdxBthSigDex, Indexer, - CtrDex, Counter, sniff, ProDex) -from keri.core.coring import (Verfer, Cigar, Signer, Salter, Saider, DigDex, - Diger, Prefixer, Cipher, Encrypter, Decrypter) -from keri.core.coring import versify, deversify, Rever, VERFULLSIZE, MINSNIFFSIZE -from keri.core.coring import generateSigners, generatePrivates -from keri.core.coring import (intToB64, intToB64b, b64ToInt, codeB64ToB2, codeB2ToB64, - B64_CHARS, Reb64, nabSextets) -from keri.help import helping +from keri import kering from keri.kering import (EmptyMaterialError, RawMaterialError, DerivationError, ShortageError, InvalidCodeSizeError, InvalidVarIndexError, - InvalidValueError, DeserializeError) -from keri.kering import Version, Versionage, VersionError -from keri.kering import (ICP_LABELS, DIP_LABELS, ROT_LABELS, DRT_LABELS, IXN_LABELS, - RPY_LABELS) -from keri.kering import (VCP_LABELS, VRT_LABELS, ISS_LABELS, BIS_LABELS, REV_LABELS, - BRV_LABELS, TSN_LABELS, CRED_TSN_LABELS) + InvalidValueError, DeserializeError, ValidationError, + InvalidVarRawSizeError, ConversionError, + SoftMaterialError, InvalidSoftError, InvalidCodeError) +from keri.kering import Version, Versionage, VersionError, Vrsn_1_0, Vrsn_2_0 +from keri.kering import Protocols, Protocolage, Ilkage, Ilks, TraitDex +from keri.help import helping +from keri.help.helping import (sceil, intToB64, intToB64b, b64ToInt, + codeB64ToB2, codeB2ToB64, + B64_CHARS, Reb64, nabSextets) +from keri import core +from keri.core import coring +from keri.core.coring import (Saids, Sadder, Tholder, Seqner, NumDex, Number, + Dater, Bexter, Texter, + TagDex, Tagger, Ilker, Traitor, Labeler, LabelDex, + Verser, Versage, ) +from keri.core.coring import Kindage, Kinds +from keri.core.coring import (Sizage, MtrDex, Matter) +from keri.core.coring import (Verfer, Cigar, Saider, DigDex, + Diger, Prefixer,) +from keri.core.coring import versify, deversify, Rever, MAXVERFULLSPAN -def test_protos(): - """ - Test protocols namedtuple instance Protos - """ +from keri.core.indexing import (Siger, Xizage, IdrDex, IdxSigDex, + IdxCrtSigDex, IdxBthSigDex, Indexer) - assert isinstance(Protos, Protocolage) - assert Protos.keri == 'KERI' - assert Protos.crel == 'CREL' - assert Protos.acdc == 'ACDC' +from keri.core.coring import MapHood, MapDom - assert 'KERI' in Protos - assert 'CREL' in Protos - assert 'ACDC' in Protos - """End Test""" -def test_prodex(): - """ - Test genera in ProDex as instance of ProtocolGenusCodex +def test_mapdom(): + """Test MapDom base dataclass""" - """ + @dataclass + class TestMapDom(MapHood): + """ + + """ + xray: str = 'X' + yankee: str = 'Y' + zulu: str = 'Z' + + def __iter__(self): # so value in dataclass not key in dataclass + return iter(astuple(self)) + + tmd = TestMapDom() + + assert 'X' in tmd + assert 'Y' in tmd + assert 'Z' in tmd + + assert tmd["xray"] == tmd.xray == 'X' + assert tmd["yankee"] == tmd.yankee == 'Y' + assert tmd["zulu"] == tmd.zulu == 'Z' + + + tmd["xray"] = "x" + assert tmd.xray == tmd["xray"] == "x" + + tmd["yankee"] = "y" + assert tmd.yankee == tmd["yankee"] == "y" + + tmd["zulu"] = "z" + assert tmd.zulu == tmd["zulu"] == "z" + + delattr(tmd, "zulu") # deletes instance attribute + assert tmd.zulu == "Z" # so returns so class attribute default value + + tmd["zulu"] = "z" + assert tmd["zulu"] == "z" + + del tmd["zulu"] # deletes instance attribute + assert tmd.zulu == "Z" # so returns so class attribute default value + + # create dynamic attribute + with pytest.raises(AttributeError): + assert tmd.alpha == None + + with pytest.raises(IndexError): + assert tmd["alpha"] == None + + tmd["alpha"] = "A" # add new attribute but without default + assert tmd.alpha == tmd["alpha"] == "A" + + del tmd["alpha"] # deletes instance attribute and no class default + + with pytest.raises(AttributeError): + assert tmd.alpha == "A" + + with pytest.raises(IndexError): + assert tmd["alpha"] == "A" + + # another dynamic attribut but delattr instead of del + with pytest.raises(AttributeError): + assert tmd.beta == None + + with pytest.raises(IndexError): + assert tmd["beta"] == None + + tmd["beta"] = "B" # add new attribute but without default + assert tmd.beta == tmd["beta"] == "B" - assert dataclasses.asdict(ProDex) == { - 'KERI': '--AAA', # KERI and ACDC Protocol Stacks share the same tables - 'ACDC': '--AAA', - } + delattr(tmd, "beta") # deletes instance attribute and no class default - assert '--AAA' in ProDex - assert ProDex.KERI == "--AAA" - assert ProDex.ACDC == "--AAA" - assert ProDex.KERI == ProDex.ACDC + with pytest.raises(AttributeError): + assert tmd.beta == "B" + + with pytest.raises(IndexError): + assert tmd["beta"] == "B" + + # attempt to delete non-existing + with pytest.raises(IndexError): + del tmd["gamma"] + + with pytest.raises(AttributeError): + delattr(tmd, "gamma") """End Test""" -def test_ilks(): - """ - Test Ilkage namedtuple instance Ilks - """ - assert Ilks == Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', - rct='rct', qry='qry', rpy='rpy', - exn='exn', pro='pro', bar='bar', - vcp='vcp', vrt='vrt', - iss='iss', rev='rev', bis='bis', brv='brv', ) - - assert isinstance(Ilks, Ilkage) - - for fld in Ilks._fields: - assert fld == getattr(Ilks, fld) - - assert 'icp' in Ilks - assert Ilks.icp == 'icp' - assert 'rot' in Ilks - assert Ilks.rot == 'rot' - assert 'ixn' in Ilks - assert Ilks.ixn == 'ixn' - assert 'dip' in Ilks - assert Ilks.dip == 'dip' - assert 'drt' in Ilks - assert Ilks.drt == 'drt' - assert 'rct' in Ilks - assert Ilks.rct == 'rct' - assert 'qry' in Ilks - assert Ilks.qry == 'qry' - assert 'rpy' in Ilks - assert Ilks.rpy == 'rpy' - assert 'exn' in Ilks - assert Ilks.exn == 'exn' - - - assert 'pro' in Ilks - assert Ilks.pro == 'pro' - assert 'bar' in Ilks - assert Ilks.bar == 'bar' - - - assert 'vcp' in Ilks - assert Ilks.vcp == 'vcp' - assert 'vrt' in Ilks - assert Ilks.vrt == 'vrt' - assert 'iss' in Ilks - assert Ilks.iss == 'iss' - assert 'rev' in Ilks - assert Ilks.rev == 'rev' - assert 'bis' in Ilks - assert Ilks.bis == 'bis' - assert 'brv' in Ilks - assert Ilks.brv == 'brv' - - """End Test """ - -def test_labels(): - """ - Test Ilkage namedtuple instance Labels - """ - assert Labels == Ilkage(icp=ICP_LABELS, rot=ROT_LABELS, ixn=IXN_LABELS, - dip=DIP_LABELS, drt=DRT_LABELS, - rct=[], qry=[], rpy=RPY_LABELS, - exn=[], pro=[], bar=[], - vcp=VCP_LABELS, vrt=VRT_LABELS, iss=ISS_LABELS, - rev=REV_LABELS, bis=BIS_LABELS, brv=BRV_LABELS) +def test_mapcodex(): + """Test MapCodex base dataclass frozen""" - assert isinstance(Labels, Ilkage) - for fld in Labels._fields: - assert isinstance(getattr(Labels, fld), list) + @dataclass(frozen=True) + class TestMapCodex(MapDom): + """ - assert Labels.icp == ICP_LABELS - assert Labels.rot == ROT_LABELS - assert Labels.ixn == IXN_LABELS - assert Labels.dip == DIP_LABELS - assert Labels.drt == DRT_LABELS - assert Labels.rct == [] - assert Labels.qry == [] - assert Labels.rpy == RPY_LABELS - assert Labels.exn == [] - assert Labels.pro == [] - assert Labels.bar == [] + """ + xray: str = 'X' + yankee: str = 'Y' + zulu: str = 'Z' - assert Labels.vcp == VCP_LABELS - assert Labels.vrt == VRT_LABELS - assert Labels.iss == ISS_LABELS - assert Labels.rev == REV_LABELS - assert Labels.bis == BIS_LABELS - assert Labels.brv == BRV_LABELS + def __iter__(self): # so value in dataclass not key in dataclass + return iter(astuple(self)) - """End Test """ + tmc = TestMapCodex() + assert 'X' in tmc + assert 'Y' in tmc + assert 'Z' in tmc + assert tmc.xray == tmc["xray"] == 'X' + assert tmc.yankee == tmc["yankee"] == 'Y' + assert tmc.zulu == tmc["zulu"] == 'Z' + with pytest.raises(IndexError): + tmc["xray"] = "x" -def test_b64_conversions(): - """ - Test Base64 conversion utility routines - """ + with pytest.raises(AttributeError): + tmc.xray = "x" - cs = intToB64(0) - assert cs == "A" - i = b64ToInt(cs) - assert i == 0 + with pytest.raises(IndexError): + del tmc["xray"] - cs = intToB64(0, l=0) - assert cs == "" - with pytest.raises(ValueError): - i = b64ToInt(cs) + with pytest.raises(AttributeError): + delattr(tmc, "xray") - cs = intToB64(None, l=0) - assert cs == "" - with pytest.raises(ValueError): - i = b64ToInt(cs) - - cs = intToB64b(0) - assert cs == b"A" - i = b64ToInt(cs) - assert i == 0 - - cs = intToB64(27) - assert cs == "b" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64b(27) - assert cs == b"b" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64(27, l=2) - assert cs == "Ab" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64b(27, l=2) - assert cs == b"Ab" - i = b64ToInt(cs) - assert i == 27 - - cs = intToB64(80) - assert cs == "BQ" - i = b64ToInt(cs) - assert i == 80 - - cs = intToB64b(80) - assert cs == b"BQ" - i = b64ToInt(cs) - assert i == 80 - - cs = intToB64(4095) - assert cs == '__' - i = b64ToInt(cs) - assert i == 4095 - - cs = intToB64b(4095) - assert cs == b'__' - i = b64ToInt(cs) - assert i == 4095 - - cs = intToB64(4096) - assert cs == 'BAA' - i = b64ToInt(cs) - assert i == 4096 - - cs = intToB64b(4096) - assert cs == b'BAA' - i = b64ToInt(cs) - assert i == 4096 - - cs = intToB64(6011) - assert cs == "Bd7" - i = b64ToInt(cs) - assert i == 6011 - - cs = intToB64b(6011) - assert cs == b"Bd7" - i = b64ToInt(cs) - assert i == 6011 - - s = "-BAC" - b = codeB64ToB2(s[:]) - assert len(b) == 3 - assert b == b'\xf8\x10\x02' - t = codeB2ToB64(b, 4) - assert t == s[:] - i = int.from_bytes(b, 'big') - assert i == 0o76010002 - i >>= 2 * (len(s) % 4) - assert i == 0o76010002 - p = nabSextets(b, 4) - assert p == b'\xf8\x10\x02' - - b = codeB64ToB2(s[:3]) - assert len(b) == 3 - assert b == b'\xf8\x10\x00' - t = codeB2ToB64(b, 3) - assert t == s[:3] - i = int.from_bytes(b, 'big') - assert i == 0o76010000 - i >>= 2 * (len(s[:3]) % 4) - assert i == 0o760100 - p = nabSextets(b, 3) - assert p == b'\xf8\x10\x00' - - b = codeB64ToB2(s[:2]) - assert len(b) == 2 - assert b == b'\xf8\x10' - t = codeB2ToB64(b, 2) - assert t == s[:2] - i = int.from_bytes(b, 'big') - assert i == 0o174020 - i >>= 2 * (len(s[:2]) % 4) - assert i == 0o7601 - p = nabSextets(b, 2) - assert p == b'\xf8\x10' - - b = codeB64ToB2(s[:1]) - assert len(b) == 1 - assert b == b'\xf8' - t = codeB2ToB64(b, 1) - assert t == s[:1] - i = int.from_bytes(b, 'big') - assert i == 0o370 - i >>= 2 * (len(s[:1]) % 4) - assert i == 0o76 - p = nabSextets(b, 1) - assert p == b'\xf8' - - assert B64_CHARS == ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', - 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', - 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_') - assert '@' not in B64_CHARS - assert 'A' in B64_CHARS - - text = b"-A-Bg-1-3-cd" - match = Reb64.match(text) - assert match - assert match is not None - - text = b'' - match = Reb64.match(text) - assert match - assert match is not None - - text = b'123#$' - match = Reb64.match(text) - assert not match - assert match is None + with pytest.raises(IndexError): + tmc["alpha"] = "A" + + with pytest.raises(AttributeError): + tmc.alpha = "A" + + # attempt to delete non-existing + with pytest.raises(IndexError): + del tmc["gamma"] + + with pytest.raises(AttributeError): + delattr(tmc, "gamma") """End Test""" -def test_matter(): +def test_matter_class(): """ - Test Matter class + Test Matter class attributes """ - assert dataclasses.asdict(MtrDex) == { + assert Matter.Codex == MtrDex + + assert Matter.Pad == '_' + + assert Matter.Codes == \ + { 'Ed25519_Seed': 'A', 'Ed25519N': 'B', 'X25519': 'C', @@ -372,6 +236,7 @@ def test_matter(): 'Label2': 'W', 'Tag3': 'X', 'Tag7': 'Y', + 'Blind': 'Z', 'Salt_128': '0A', 'Ed25519_Sig': '0B', 'ECDSA_256k1_Sig': '0C', @@ -385,6 +250,8 @@ def test_matter(): 'Tag2': '0K', 'Tag5': '0L', 'Tag6': '0M', + 'Tag9': '0N', + 'Tag10': '0O', 'ECDSA_256k1N': '1AAA', 'ECDSA_256k1': '1AAB', 'Ed448N': '1AAC', @@ -396,10 +263,15 @@ def test_matter(): 'ECDSA_256r1N': '1AAI', 'ECDSA_256r1': '1AAJ', 'Null': '1AAK', - 'Yes': '1AAL', - 'No': '1AAM', - 'TBD1': '2AAA', - 'TBD2': '3AAA', + 'No': '1AAL', + 'Yes': '1AAM', + 'Tag8': '1AAN', + 'TBD0S': '1__-', + 'TBD0': '1___', + 'TBD1S': '2__-', + 'TBD1': '2___', + 'TBD2S': '3__-', + 'TBD2': '3___', 'StrB64_L0': '4A', 'StrB64_L1': '5A', 'StrB64_L2': '6A', @@ -424,16 +296,109 @@ def test_matter(): 'X25519_Cipher_QB64_Big_L0': '7AAD', 'X25519_Cipher_QB64_Big_L1': '8AAD', 'X25519_Cipher_QB64_Big_L2': '9AAD', - 'X25519_Cipher_QB2_L0': '4D', - 'X25519_Cipher_QB2_L1': '5D', - 'X25519_Cipher_QB2_L2': '6D', - 'X25519_Cipher_QB2_Big_L0': '7AAD', - 'X25519_Cipher_QB2_Big_L1': '8AAD', - 'X25519_Cipher_QB2_Big_L2': '9AAD' + 'X25519_Cipher_QB2_L0': '4E', + 'X25519_Cipher_QB2_L1': '5E', + 'X25519_Cipher_QB2_L2': '6E', + 'X25519_Cipher_QB2_Big_L0': '7AAE', + 'X25519_Cipher_QB2_Big_L1': '8AAE', + 'X25519_Cipher_QB2_Big_L2': '9AAE' } + assert Matter.Names == \ + { + 'A': 'Ed25519_Seed', + 'B': 'Ed25519N', + 'C': 'X25519', + 'D': 'Ed25519', + 'E': 'Blake3_256', + 'F': 'Blake2b_256', + 'G': 'Blake2s_256', + 'H': 'SHA3_256', + 'I': 'SHA2_256', + 'J': 'ECDSA_256k1_Seed', + 'K': 'Ed448_Seed', + 'L': 'X448', + 'M': 'Short', + 'N': 'Big', + 'O': 'X25519_Private', + 'P': 'X25519_Cipher_Seed', + 'Q': 'ECDSA_256r1_Seed', + 'R': 'Tall', + 'S': 'Large', + 'T': 'Great', + 'U': 'Vast', + 'V': 'Label1', + 'W': 'Label2', + 'X': 'Tag3', + 'Y': 'Tag7', + 'Z': 'Blind', + '0A': 'Salt_128', + '0B': 'Ed25519_Sig', + '0C': 'ECDSA_256k1_Sig', + '0D': 'Blake3_512', + '0E': 'Blake2b_512', + '0F': 'SHA3_512', + '0G': 'SHA2_512', + '0H': 'Long', + '0I': 'ECDSA_256r1_Sig', + '0J': 'Tag1', + '0K': 'Tag2', + '0L': 'Tag5', + '0M': 'Tag6', + '0N': 'Tag9', + '0O': 'Tag10', + '1AAA': 'ECDSA_256k1N', + '1AAB': 'ECDSA_256k1', + '1AAC': 'Ed448N', + '1AAD': 'Ed448', + '1AAE': 'Ed448_Sig', + '1AAF': 'Tag4', + '1AAG': 'DateTime', + '1AAH': 'X25519_Cipher_Salt', + '1AAI': 'ECDSA_256r1N', + '1AAJ': 'ECDSA_256r1', + '1AAK': 'Null', + '1AAL': 'No', + '1AAM': 'Yes', + '1AAN': 'Tag8', + '1__-': 'TBD0S', + '1___': 'TBD0', + '2__-': 'TBD1S', + '2___': 'TBD1', + '3__-': 'TBD2S', + '3___': 'TBD2', + '4A': 'StrB64_L0', + '5A': 'StrB64_L1', + '6A': 'StrB64_L2', + '7AAA': 'StrB64_Big_L0', + '8AAA': 'StrB64_Big_L1', + '9AAA': 'StrB64_Big_L2', + '4B': 'Bytes_L0', + '5B': 'Bytes_L1', + '6B': 'Bytes_L2', + '7AAB': 'Bytes_Big_L0', + '8AAB': 'Bytes_Big_L1', + '9AAB': 'Bytes_Big_L2', + '4C': 'X25519_Cipher_L0', + '5C': 'X25519_Cipher_L1', + '6C': 'X25519_Cipher_L2', + '7AAC': 'X25519_Cipher_Big_L0', + '8AAC': 'X25519_Cipher_Big_L1', + '9AAC': 'X25519_Cipher_Big_L2', + '4D': 'X25519_Cipher_QB64_L0', + '5D': 'X25519_Cipher_QB64_L1', + '6D': 'X25519_Cipher_QB64_L2', + '7AAD': 'X25519_Cipher_QB64_Big_L0', + '8AAD': 'X25519_Cipher_QB64_Big_L1', + '9AAD': 'X25519_Cipher_QB64_Big_L2', + '4E': 'X25519_Cipher_QB2_L0', + '5E': 'X25519_Cipher_QB2_L1', + '6E': 'X25519_Cipher_QB2_L2', + '7AAE': 'X25519_Cipher_QB2_Big_L0', + '8AAE': 'X25519_Cipher_QB2_Big_L1', + '9AAE': 'X25519_Cipher_QB2_Big_L2' + } - assert Matter.Codex == MtrDex # first character of code with hard size of code assert Matter.Hards == { @@ -448,111 +413,193 @@ def test_matter(): } # Codes table with sizes of code (hard) and full primitive material - assert Matter.Sizes == { - 'A': Sizage(hs=1, ss=0, fs=44, ls=0), - 'B': Sizage(hs=1, ss=0, fs=44, ls=0), - 'C': Sizage(hs=1, ss=0, fs=44, ls=0), - 'D': Sizage(hs=1, ss=0, fs=44, ls=0), - 'E': Sizage(hs=1, ss=0, fs=44, ls=0), - 'F': Sizage(hs=1, ss=0, fs=44, ls=0), - 'G': Sizage(hs=1, ss=0, fs=44, ls=0), - 'H': Sizage(hs=1, ss=0, fs=44, ls=0), - 'I': Sizage(hs=1, ss=0, fs=44, ls=0), - 'J': Sizage(hs=1, ss=0, fs=44, ls=0), - 'K': Sizage(hs=1, ss=0, fs=76, ls=0), - 'L': Sizage(hs=1, ss=0, fs=76, ls=0), - 'M': Sizage(hs=1, ss=0, fs=4, ls=0), - 'N': Sizage(hs=1, ss=0, fs=12, ls=0), - 'O': Sizage(hs=1, ss=0, fs=44, ls=0), - 'P': Sizage(hs=1, ss=0, fs=124, ls=0), - 'Q': Sizage(hs=1, ss=0, fs=44, ls=0), - 'R': Sizage(hs=1, ss=0, fs=8, ls=0), - 'S': Sizage(hs=1, ss=0, fs=16, ls=0), - 'T': Sizage(hs=1, ss=0, fs=20, ls=0), - 'U': Sizage(hs=1, ss=0, fs=24, ls=0), - 'V': Sizage(hs=1, ss=0, fs=4, ls=1), - 'W': Sizage(hs=1, ss=0, fs=4, ls=0), - 'X': Sizage(hs=1, ss=0, fs=4, ls=0), - 'Y': Sizage(hs=1, ss=0, fs=8, ls=0), - '0A': Sizage(hs=2, ss=0, fs=24, ls=0), - '0B': Sizage(hs=2, ss=0, fs=88, ls=0), - '0C': Sizage(hs=2, ss=0, fs=88, ls=0), - '0D': Sizage(hs=2, ss=0, fs=88, ls=0), - '0E': Sizage(hs=2, ss=0, fs=88, ls=0), - '0F': Sizage(hs=2, ss=0, fs=88, ls=0), - '0G': Sizage(hs=2, ss=0, fs=88, ls=0), - '0H': Sizage(hs=2, ss=0, fs=8, ls=0), - '0I': Sizage(hs=2, ss=0, fs=88, ls=0), - '0J': Sizage(hs=2, ss=0, fs=4, ls=0), - '0K': Sizage(hs=2, ss=0, fs=4, ls=0), - '0L': Sizage(hs=2, ss=0, fs=8, ls=0), - '0M': Sizage(hs=2, ss=0, fs=8, ls=0), - '1AAA': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAB': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAC': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAD': Sizage(hs=4, ss=0, fs=80, ls=0), - '1AAE': Sizage(hs=4, ss=0, fs=56, ls=0), - '1AAF': Sizage(hs=4, ss=0, fs=8, ls=0), - '1AAG': Sizage(hs=4, ss=0, fs=36, ls=0), - '1AAH': Sizage(hs=4, ss=0, fs=100, ls=0), - '1AAI': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAJ': Sizage(hs=4, ss=0, fs=48, ls=0), - '1AAK': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAL': Sizage(hs=4, ss=0, fs=4, ls=0), - '1AAM': Sizage(hs=4, ss=0, fs=4, ls=0), - '2AAA': Sizage(hs=4, ss=0, fs=8, ls=1), - '3AAA': Sizage(hs=4, ss=0, fs=8, ls=2), - '4A': Sizage(hs=2, ss=2, fs=None, ls=0), - '5A': Sizage(hs=2, ss=2, fs=None, ls=1), - '6A': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAA': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAA': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAA': Sizage(hs=4, ss=4, fs=None, ls=2), - '4B': Sizage(hs=2, ss=2, fs=None, ls=0), - '5B': Sizage(hs=2, ss=2, fs=None, ls=1), - '6B': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAB': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAB': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAB': Sizage(hs=4, ss=4, fs=None, ls=2), - '4C': Sizage(hs=2, ss=2, fs=None, ls=0), - '5C': Sizage(hs=2, ss=2, fs=None, ls=1), - '6C': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAC': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAC': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAC': Sizage(hs=4, ss=4, fs=None, ls=2), - '4D': Sizage(hs=2, ss=2, fs=None, ls=0), - '5D': Sizage(hs=2, ss=2, fs=None, ls=1), - '6D': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAD': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAD': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAD': Sizage(hs=4, ss=4, fs=None, ls=2), - '4E': Sizage(hs=2, ss=2, fs=None, ls=0), - '5E': Sizage(hs=2, ss=2, fs=None, ls=1), - '6E': Sizage(hs=2, ss=2, fs=None, ls=2), - '7AAE': Sizage(hs=4, ss=4, fs=None, ls=0), - '8AAE': Sizage(hs=4, ss=4, fs=None, ls=1), - '9AAE': Sizage(hs=4, ss=4, fs=None, ls=2) + assert Matter.Sizes == \ + { + 'A': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'B': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'C': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'D': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'E': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'F': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'G': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'H': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'I': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'J': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'K': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'L': Sizage(hs=1, ss=0, xs=0, fs=76, ls=0), + 'M': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'N': Sizage(hs=1, ss=0, xs=0, fs=12, ls=0), + 'O': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'P': Sizage(hs=1, ss=0, xs=0, fs=124, ls=0), + 'Q': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + 'R': Sizage(hs=1, ss=0, xs=0, fs=8, ls=0), + 'S': Sizage(hs=1, ss=0, xs=0, fs=16, ls=0), + 'T': Sizage(hs=1, ss=0, xs=0, fs=20, ls=0), + 'U': Sizage(hs=1, ss=0, xs=0, fs=24, ls=0), + 'V': Sizage(hs=1, ss=0, xs=0, fs=4, ls=1), + 'W': Sizage(hs=1, ss=0, xs=0, fs=4, ls=0), + 'X': Sizage(hs=1, ss=3, xs=0, fs=4, ls=0), + 'Y': Sizage(hs=1, ss=7, xs=0, fs=8, ls=0), + 'Z': Sizage(hs=1, ss=0, xs=0, fs=44, ls=0), + '0A': Sizage(hs=2, ss=0, xs=0, fs=24, ls=0), + '0B': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0C': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0D': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0E': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0F': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0G': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0H': Sizage(hs=2, ss=0, xs=0, fs=8, ls=0), + '0I': Sizage(hs=2, ss=0, xs=0, fs=88, ls=0), + '0J': Sizage(hs=2, ss=2, xs=1, fs=4, ls=0), + '0K': Sizage(hs=2, ss=2, xs=0, fs=4, ls=0), + '0L': Sizage(hs=2, ss=6, xs=1, fs=8, ls=0), + '0M': Sizage(hs=2, ss=6, xs=0, fs=8, ls=0), + '0N': Sizage(hs=2, ss=10, xs=1, fs=12, ls=0), + '0O': Sizage(hs=2, ss=10, xs=0, fs=12, ls=0), + '1AAA': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAB': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAC': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAD': Sizage(hs=4, ss=0, xs=0, fs=80, ls=0), + '1AAE': Sizage(hs=4, ss=0, xs=0, fs=56, ls=0), + '1AAF': Sizage(hs=4, ss=4, xs=0, fs=8, ls=0), + '1AAG': Sizage(hs=4, ss=0, xs=0, fs=36, ls=0), + '1AAH': Sizage(hs=4, ss=0, xs=0, fs=100, ls=0), + '1AAI': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAJ': Sizage(hs=4, ss=0, xs=0, fs=48, ls=0), + '1AAK': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAL': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAM': Sizage(hs=4, ss=0, xs=0, fs=4, ls=0), + '1AAN': Sizage(hs=4, ss=8, xs=0, fs=12, ls=0), + '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), + '1___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=0), + '2__-': Sizage(hs=4, ss=2, xs=1, fs=12, ls=1), + '2___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=1), + '3__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=2), + '3___': Sizage(hs=4, ss=0, xs=0, fs=8, ls=2), + '4A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6A': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAA': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6B': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAB': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6C': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAC': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6D': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAD': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2), + '4E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=0), + '5E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=1), + '6E': Sizage(hs=2, ss=2, xs=0, fs=None, ls=2), + '7AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=0), + '8AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=1), + '9AAE': Sizage(hs=4, ss=4, xs=0, fs=None, ls=2) } + assert Matter.Sizes['A'].hs == 1 # hard size assert Matter.Sizes['A'].ss == 0 # soft size + assert Matter.Sizes['A'].xs == 0 # xtra size assert Matter.Sizes['A'].fs == 44 # full size assert Matter.Sizes['A'].ls == 0 # lead size + + # verify all Codes + for code, val in Matter.Sizes.items(): # hard code + hs = val.hs + ss = val.ss + xs = val.xs + fs = val.fs + ls = val.ls + cs = hs + ss + + assert (isinstance(hs, int) and isinstance(ss, int) and + isinstance(ls, int)) + assert hs > 0 and ss >= 0 and xs in (0, 1, 2) and ls in (0, 1, 2) + assert len(code) == hs + + if fs is None: # variable sized + assert ss > 0 and xs == 0 and not (cs % 4) # full code is 24 bit aligned + # assumes that Matter methods also ensure (ls + rs) % 3 == 0 i.e. + # variable raw with lead is 24 bit aligned, where rs is raw size. + assert code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex + + if code[0] in coring.SmallVrzDex: # small variable sized code + assert hs == 2 and ss == 2 and fs is None + assert code[0] == astuple(coring.SmallVrzDex)[ls] + if code[0] in '4': + assert ls == 0 + elif code[0] in '5': + assert ls == 1 + elif code[0] in '6': + assert ls == 2 + else: + assert False + + elif code[0] in coring.LargeVrzDex: # large veriable sized code + assert val.hs == 4 and val.ss == 4 and val.fs is None + assert code[0] == astuple(coring.LargeVrzDex)[ls] + if code[0] in '7': + assert ls == 0 + elif code[0] in '8': + assert ls == 1 + elif code[0] in '9': + assert ls == 2 + else: + assert False + + else: + assert False + + else: # fixed size + assert not (code[0] in coring.SmallVrzDex or code[0] in coring.LargeVrzDex) + assert isinstance(fs, int) and fs > 0 and not fs % 4 + assert fs >= cs + assert xs <= ss # xs must be zero if ss is + assert cs % 4 != 3 # prevent ambiguous conversion + if ss > 0 and fs == cs: # special soft value with raw empty + assert ls == 0 # no lead + assert Matter._rawSize(code) == 0 + assert xs < ss # soft must not be empty, not all prepad + + # verify correct sizes given raw size. Assumes properties above + rs = ((fs - cs) * 3 // 4) - ls # raw size bytes sans lead + assert sceil((rs + ls) * 4 / 3) + cs == fs # sextets add up + ps = (3 - ((rs + ls) % 3)) % 3 # net pad size given raw with lead + assert ps == (cs % 4) # ensure correct midpad zero bits for cs + + if code[0] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz': + assert len(code) == 1 + elif code[0] in '0': + assert len(code) == 2 + elif code[0] in '1': + assert len(code) == 4 and ls == 0 + elif code[0] in '2': + assert len(code) == 4 and ls == 1 + elif code[0] in '3': + assert len(code) == 4 and ls == 2 + else: + assert code[0] not in '456789-_' # count or op code + + + # Test .Hards # verify first hs Sizes matches hs in Codes for same first char for ckey in Matter.Sizes.keys(): assert Matter.Hards[ckey[0]] == Matter.Sizes[ckey].hs - # verify all Codes have ss == 0 and not fs % 4 and hs > 0 and fs > hs - # if fs is not None else not (hs + ss) % 4 - for val in Matter.Sizes.values(): - if val.fs is not None: - assert val.ss == 0 and not val.fs % 4 and val.hs > 0 and val.fs >= (val.hs + val.ss) - else: - assert not (val.hs + val.ss) % 4 - - - # Bizes maps bytes of sextet of decoded first character of code with hard size of code + # Test .Bards # verify equivalents of items for Sizes and Bizes for skey, sval in Matter.Hards.items(): ckey = codeB64ToB2(skey) @@ -560,6 +607,17 @@ def test_matter(): assert Matter._rawSize(MtrDex.Ed25519) == 32 assert Matter._leadSize(MtrDex.Ed25519) == 0 + assert Matter._xtraSize(MtrDex.Ed25519) == 0 + assert not Matter._special(MtrDex.Ed25519) + assert Matter._special(MtrDex.Tag3) + + + +def test_matter(): + """ + Test Matter instances + """ + # verkey, sigkey = pysodium.crypto_sign_keypair() verkey = b'iN\x89Gi\xe6\xc3&~\x8bG|%\x90(L\xd6G\xddB\xef`\x07\xd2T\xfc\xe1\xcd.\x9b\xe4#' @@ -581,7 +639,9 @@ def test_matter(): # test from raw matter = Matter(raw=verkey) # default code is MtrDex.Ed25519N assert matter.raw == verkey - assert matter.code == MtrDex.Ed25519N + assert matter.code == MtrDex.Ed25519N == matter.hard + assert matter.name == 'Ed25519N' + assert matter.soft == "" assert matter.both == MtrDex.Ed25519N assert matter.size == None assert matter.fullSize == 44 @@ -596,10 +656,14 @@ def test_matter(): assert matter.transferable == False assert matter.digestive == False assert matter.prefixive == True + assert not matter.special + assert matter.composable + # test round trip assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) + assert matter.composable # Test from qb64b matter = Matter(qb64b=prefixb) @@ -623,15 +687,17 @@ def test_matter(): # test non-zero pad bits in qb64 init ps == 1 badprefix1 = 'B_AAY2RlZmdoaWprbG1ub3BxcnN0dXYwMTIzNDU2Nzg5' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badprefix1) - assert str(ex.value) == "Non zeroed prepad bits = 110000 in b'_'." + #assert str(ex.value) == "Non zeroed prepad bits = 110000 in b'_'." + assert str(ex.value) == 'Nonzero midpad bytes=0x03.' # test non-zero pad bits in qb64 init ps == 2 badprefix2 = '0A_wMTIzNDU2Nzg5YWJjZGVm' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badprefix2) - assert str(ex.value) == "Non zeroed prepad bits = 111100 in b'_'." + #assert str(ex.value) == "Non zeroed prepad bits = 111100 in b'_'." + assert str(ex.value) == 'Nonzero midpad bytes=0x000f.' # test truncates extra bytes from qb64 parameter longprefix = prefix + "ABCD" # extra bytes in size @@ -656,15 +722,17 @@ def test_matter(): # test non-zero pad bits in qb2 init ps ==1 badprebin1 = decodeB64(badprefix1) # b'\x07\xf0\x00cdefghijklmnopqrstuv0123456789' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badprebin1) - assert str(ex.value) == 'Non zeroed pad bits = 00000011 in 0x07.' + #assert str(ex.value) == 'Non zeroed pad bits = 00000011 in 0x07.' + assert str(ex.value) == 'Nonzero code mid pad bits=0b11.' # test non-zero pad bits in qb2 init ps ==2 badprebin2 = decodeB64(badprefix2) # b'\xd0\x0f\xf0123456789abcdef' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badprebin2) - assert str(ex.value) == 'Non zeroed pad bits = 00001111 in 0x0f.' + #assert str(ex.value) == 'Non zeroed pad bits = 00001111 in 0x0f.' + assert str(ex.value) == 'Nonzero code mid pad bits=0b1111.' # test raises ShortageError if not enough bytes in qb2 parameter @@ -800,18 +868,94 @@ def test_matter(): assert matter.prefixive == True assert ims == extra # stripped not include extra + # test fixed size with leader 0 + # TBD0 = '1___' # Testing purposes only fixed with lead size 0 + + code = MtrDex.TBD0 # '1___' + assert Matter._rawSize(code) == 3 + assert Matter._leadSize(code) == 0 + raw = b'abc' + qb64 = '1___YWJj' # + qb2 = decodeB64(qb64) + matter = Matter(raw=raw, code=code) + assert matter.raw == raw + assert matter.code == code + assert matter.both == code + assert matter.size == None + assert matter.fullSize == 8 + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.transferable == True + assert matter.digestive == False + assert matter.prefixive == False + assert not matter.special + assert matter.composable + + assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") + assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) + + matter._exfil(qb64.encode("utf-8")) + assert matter.code == code + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + + matter = Matter(qb64b=qb64.encode("utf-8")) + assert matter.code == code + assert matter.raw == raw + + matter = Matter(qb64=qb64) + assert matter.code == code + assert matter.raw == raw + + matter = Matter(qb64=qb64.encode("utf-8")) # works for either + assert matter.code == code + assert matter.raw == raw + + # Test ._bexfil + matter._bexfil(qb2) + assert matter.raw == raw + assert matter.code == code + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + + matter = Matter(qb2=qb2) + assert matter.code == code + assert matter.raw == raw + assert matter.qb64b == qb64.encode("utf-8") + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.transferable == True + assert matter.digestive == False + + matter = Matter(raw=raw, code=code) + assert matter.raw == raw + assert matter.code == code + assert matter.both == code + assert matter.size == None + assert matter.fullSize == 8 + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.transferable == True + assert matter.digestive == False + assert matter.prefixive == False + + # Can't have bad pad because cs % 4 == 0 + # Can't habe bad lead because ls ==0 + # test fix sized with leader 1 - # TBD1 = '2AAA' # Testing purposes only fixed with lead size 1 + # TBD1 = '2___' # Testing purposes only fixed with lead size 1 - code = MtrDex.TBD1 # '2AAA' + code = MtrDex.TBD1 # '2___' assert Matter._rawSize(code) == 2 assert Matter._leadSize(code) == 1 raw = b'ab' - qb64 = '2AAAAGFi' # '2AAA' + encodeB64(b'\x00ab').decode("utf-8") - qb2 = decodeB64(qb64) # b'\xd8\x00\x00\x00ab' + qb64 = '2___AGFi' # '2___' + encodeB64(b'\x00ab').decode("utf-8") + qb2 = decodeB64(qb64) matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code + assert matter.name == 'TBD1' assert matter.both == code assert matter.size == None assert matter.fullSize == 8 @@ -820,6 +964,8 @@ def test_matter(): assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) @@ -871,26 +1017,28 @@ def test_matter(): assert matter.prefixive == False # test with bad pad or lead - badqb64 = '2AAA_2Fi' # '2AAA' + encodeB64(b'\xffab').decode("utf-8") + badqb64 = '2____2Fi' # '2___' + encodeB64(b'\xffab').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xd8\x00\x00\xffab' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xff.' # test fix sized with leader 2 - # TBD2 = '3AAA' # Testing purposes only of fixed with lead size 2 + # TBD2 = '3___' # Testing purposes only of fixed with lead size 2 code = MtrDex.TBD2 # '3AAA' assert Matter._rawSize(code) == 1 assert Matter._leadSize(code) == 2 raw = b'z' - qb64 = '3AAAAAB6' - qb2 = b'\xdc\x00\x00\x00\x00z' + qb64 = '3___AAB6' + qb2 = decodeB64(qb64) matter = Matter(raw=raw, code=code) assert matter.raw == raw assert matter.code == code @@ -902,6 +1050,8 @@ def test_matter(): assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable assert matter.qb64 == encodeB64(matter.qb2).decode("utf-8") assert matter.qb2 == decodeB64(matter.qb64.encode("utf-8")) @@ -940,16 +1090,18 @@ def test_matter(): assert matter.prefixive == False # test with bad pad or lead - badqb64 = '3AAA__96' # '3AAA' + encodeB64(b'\xff\xffz').decode("utf-8") + badqb64 = '3_____96' # '3AAA' + encodeB64(b'\xff\xffz').decode("utf-8") badqb2 = decodeB64(badqb64) #b'\xdc\x00\x00\xff\xffz' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xffff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xffff.' # test variable sized with leader 1 code = MtrDex.Bytes_L1 @@ -958,13 +1110,15 @@ def test_matter(): assert Matter._leadSize(code) == 1 raw = b'abcde' # 5 bytes two triplets with lead 1 both = '5BAC' # full code both hard and soft parts two quadlets/triplets + soft = 'AC' qb64 = '5BACAGFiY2Rl' qb2 = b'\xe4\x10\x02\x00abcde' matter = Matter(raw=raw, code=code) assert matter.raw == raw - assert matter.code == code - assert matter.both == both + assert matter.code == code == matter.hard assert matter.size == 2 # quadlets + assert matter.soft == soft + assert matter.both == both assert matter.fullSize == 12 # chars assert matter.qb64 == qb64 assert matter.qb2 == qb2 @@ -1029,13 +1183,15 @@ def test_matter(): badqb64 = '5BAC_2FiY2Rl' # '5BAC' + encodeB64(b'\xffabcde').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xe4\x10\x02\xffabcde' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + #assert str(ex.value) == 'Non zeroed lead byte = 0xff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xff.' # test variable sized with leader 1 with code replacement code0 = MtrDex.Bytes_L0 # use leader 0 code but with lead size 1 raw @@ -1162,13 +1318,15 @@ def test_matter(): badqb64 = '6BAC__9hYmNk' # '5BAC' + encodeB64(b'\xff\xffabcd').decode("utf-8") badqb2 = decodeB64(badqb64) # b'\xe8\x10\x02\xff\xffabcd' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb64=badqb64) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Nonzero midpad bytes=0xffff.' - with pytest.raises(ValueError) as ex: + with pytest.raises(ConversionError) as ex: matter = Matter(qb2=badqb2) - assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + #assert str(ex.value) == 'Non zeroed lead bytes = 0xffff.' + assert str(ex.value) == 'Nonzero lead midpad bytes=0xffff.' # test variable sized with leader 2 with code replacement code0 = MtrDex.Bytes_L0 # use leader 0 code but with lead size 2 raw @@ -1593,1230 +1751,430 @@ def test_matter(): assert matter.digestive == False assert matter.prefixive == False - # test Tag4 - #val = int("F89CFF", 16) - #assert val == 16293119 - #raw = val.to_bytes(3, 'big') - #assert raw == b'\xf8\x9c\xff' - raw = b'hio' - cs = len(MtrDex.Tag4) - assert cs == 4 - ps = cs % 4 - assert ps == 0 - txt = encodeB64(bytes([0]*ps) + raw) - #assert txt == b'-Jz_' - assert txt == b'aGlv' - qb64b = MtrDex.Tag4.encode("utf-8") + txt[ps:] - #assert qb64b == b'1AAF-Jz_' - assert qb64b == b'1AAFaGlv' - qb64 = qb64b.decode("utf-8") + # test Label1 + code = MtrDex.Label1 + raw = b'*' + qb64 = 'VAAq' + qb64b = qb64.encode("utf-8") qb2 = decodeB64(qb64b) - assert qb2 == b'\xd4\x00\x05hio' - #assert qb2 == b'\xd4\x00\x05\xf8\x9c\xff' - bs = ceil((cs * 3) / 4) - assert qb2[bs:] == raw # stable value in qb2 - assert encodeB64(qb2) == qb64b - matter = Matter(raw=raw, code=MtrDex.Tag4) + matter = Matter(raw=raw, code=code) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64b=qb64b) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64=qb64) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b - assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb2=qb2) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False - - # test Tag4 as chars - txt = b'icp_' - raw = decodeB64(txt) - assert raw == b'\x89\xca\x7f' - val = int.from_bytes(raw, 'big') - assert val == 9030271 - cs = len(MtrDex.Tag4) - assert cs == 4 - ps = cs % 4 - assert ps == 0 - txt = encodeB64(bytes([0]*ps) + raw) - qb64b = MtrDex.Tag4.encode("utf-8") + txt - assert qb64b == b'1AAFicp_' - qb64 = qb64b.decode("utf-8") + assert not matter.special + assert matter.composable + + # test Label2 + code = MtrDex.Label2 + raw = b'@&' + qb64 = 'WEAm' + qb64b = qb64.encode("utf-8") qb2 = decodeB64(qb64b) - assert qb2 == b'\xd4\x00\x05\x89\xca\x7f' - bs = ceil((cs * 3) / 4) - assert qb2[bs:] == raw # stable value in qb2 - assert encodeB64(qb2) == qb64b - matter = Matter(raw=raw, code=MtrDex.Tag4) + matter = Matter(raw=raw, code=code) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64b=qb64b) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb64=qb64) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code == code assert matter.qb64 == qb64 assert matter.qb64b == qb64b - assert matter.qb2 == qb2 assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable matter = Matter(qb2=qb2) assert matter.raw == raw - assert matter.code == MtrDex.Tag4 + assert matter.code ==code assert matter.qb64 == qb64 assert matter.qb64b == qb64b assert matter.qb2 == qb2 - bs = ceil((len(matter.code) * 3) / 4) - assert matter.qb2[bs:] == matter.raw assert matter.transferable == True assert matter.digestive == False assert matter.prefixive == False + assert not matter.special + assert matter.composable - """ Done Test """ + """ Done Test """ -def test_indexer(): +def test_matter_special(): """ - Test Indexer class + Test Matter instances using code with special soft values """ - assert Indexer.Codex == IdrDex - - assert dataclasses.asdict(IdrDex) == { - 'Ed25519_Sig': 'A', - 'Ed25519_Crt_Sig': 'B', - 'ECDSA_256k1_Sig': 'C', - 'ECDSA_256k1_Crt_Sig': 'D', - 'ECDSA_256r1_Sig': 'E', - 'ECDSA_256r1_Crt_Sig': 'F', - 'Ed448_Sig': '0A', - 'Ed448_Crt_Sig': '0B', - 'Ed25519_Big_Sig': '2A', - 'Ed25519_Big_Crt_Sig': '2B', - 'ECDSA_256k1_Big_Sig': '2C', - 'ECDSA_256k1_Big_Crt_Sig': '2D', - 'ECDSA_256r1_Big_Sig': '2E', - 'ECDSA_256r1_Big_Crt_Sig': '2F', - 'Ed448_Big_Sig': '3A', - 'Ed448_Big_Crt_Sig': '3B', - 'TBD0': '0z', - 'TBD1': '1z', - 'TBD4': '4z', - } + # test Tag3 + code = MtrDex.Tag3 + soft = 'icp' + qb64 = 'Xicp' + qb2 = b"^')" + raw = b'' + + matter = Matter(code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - assert IdrDex.Ed25519_Sig == 'A' - assert IdrDex.Ed25519_Crt_Sig == 'B' - assert IdrDex.ECDSA_256k1_Sig == 'C' - assert IdrDex.ECDSA_256k1_Crt_Sig == 'D' - assert IdrDex.ECDSA_256r1_Sig == 'E' - assert IdrDex.ECDSA_256r1_Crt_Sig == 'F' - assert IdrDex.Ed448_Sig == '0A' - assert IdrDex.Ed448_Crt_Sig == '0B' - assert IdrDex.Ed25519_Big_Sig == '2A' - assert IdrDex.Ed25519_Big_Crt_Sig == '2B' - assert IdrDex.ECDSA_256k1_Big_Sig == '2C' - assert IdrDex.ECDSA_256k1_Big_Crt_Sig == '2D' - assert IdrDex.ECDSA_256r1_Big_Sig == '2E' - assert IdrDex.ECDSA_256r1_Big_Crt_Sig == '2F' - assert IdrDex.Ed448_Big_Sig == '3A' - assert IdrDex.Ed448_Big_Crt_Sig == '3B' - assert IdrDex.TBD0 == '0z' - assert IdrDex.TBD1 == '1z' - assert IdrDex.TBD4 == '4z' - - assert dataclasses.asdict(IdxSigDex) == { - 'Ed25519_Sig': 'A', - 'Ed25519_Crt_Sig': 'B', - 'ECDSA_256k1_Sig': 'C', - 'ECDSA_256k1_Crt_Sig': 'D', - 'ECDSA_256r1_Sig': 'E', - 'ECDSA_256r1_Crt_Sig': 'F', - 'Ed448_Sig': '0A', - 'Ed448_Crt_Sig': '0B', - 'Ed25519_Big_Sig': '2A', - 'Ed25519_Big_Crt_Sig': '2B', - 'ECDSA_256k1_Big_Sig': '2C', - 'ECDSA_256k1_Big_Crt_Sig': '2D', - 'ECDSA_256r1_Big_Sig': '2E', - 'ECDSA_256r1_Big_Crt_Sig': '2F', - 'Ed448_Big_Sig': '3A', - 'Ed448_Big_Crt_Sig': '3B', - } + code = matter.code + soft = matter.soft + qb2 = matter.qb2 + qb64 = matter.qb64 - assert IdxSigDex.Ed25519_Sig == 'A' - assert IdxSigDex.Ed25519_Crt_Sig == 'B' - assert IdxSigDex.ECDSA_256k1_Sig == 'C' - assert IdxSigDex.ECDSA_256k1_Crt_Sig == 'D' - assert IdxSigDex.ECDSA_256r1_Sig == 'E' - assert IdxSigDex.ECDSA_256r1_Crt_Sig == 'F' - assert IdxSigDex.Ed448_Sig == '0A' - assert IdxSigDex.Ed448_Crt_Sig == '0B' - assert IdxSigDex.Ed25519_Big_Sig == '2A' - assert IdxSigDex.Ed25519_Big_Crt_Sig == '2B' - assert IdxSigDex.ECDSA_256k1_Big_Sig == '2C' - assert IdxSigDex.ECDSA_256k1_Big_Crt_Sig == '2D' - assert IdxSigDex.ECDSA_256r1_Big_Sig == '2E' - assert IdxSigDex.ECDSA_256r1_Big_Crt_Sig == '2F' - assert IdxSigDex.Ed448_Big_Sig == '3A' - assert IdxSigDex.Ed448_Big_Crt_Sig == '3B' - - - assert dataclasses.asdict(IdxCrtSigDex) == { - 'Ed25519_Crt_Sig': 'B', - 'ECDSA_256k1_Crt_Sig': 'D', - 'ECDSA_256r1_Crt_Sig': 'F', - 'Ed448_Crt_Sig': '0B', - 'Ed25519_Big_Crt_Sig': '2B', - 'ECDSA_256k1_Big_Crt_Sig': '2D', - 'ECDSA_256r1_Big_Crt_Sig': '2F', - 'Ed448_Big_Crt_Sig': '3B', - } + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - assert IdxCrtSigDex.Ed25519_Crt_Sig == 'B' - assert IdxCrtSigDex.ECDSA_256k1_Crt_Sig == 'D' - assert IdxCrtSigDex.ECDSA_256r1_Crt_Sig == 'F' - assert IdxCrtSigDex.Ed448_Crt_Sig == '0B' - assert IdxCrtSigDex.Ed25519_Big_Crt_Sig == '2B' - assert IdxCrtSigDex.ECDSA_256k1_Big_Crt_Sig == '2D' - assert IdxCrtSigDex.ECDSA_256r1_Big_Crt_Sig == '2F' - assert IdxCrtSigDex.Ed448_Big_Crt_Sig == '3B' - - - assert dataclasses.asdict(IdxBthSigDex) == { - 'Ed25519_Sig': 'A', - 'ECDSA_256k1_Sig': 'C', - 'ECDSA_256r1_Sig': 'E', - 'Ed448_Sig': '0A', - 'Ed25519_Big_Sig': '2A', - 'ECDSA_256k1_Big_Sig': '2C', - 'ECDSA_256r1_Big_Sig': '2E', - 'Ed448_Big_Sig': '3A', - } + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # Test corner conditions + # Empty raw + matter = Matter(raw=b'', code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + #non empty raw ignored since code special, forces empty raw + badraw = b'abcdefg' + matter = Matter(raw=badraw, code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - assert IdxBthSigDex.Ed25519_Sig == 'A' - assert IdxBthSigDex.ECDSA_256k1_Sig == 'C' - assert IdxBthSigDex.ECDSA_256r1_Sig == 'E' - assert IdxBthSigDex.Ed448_Sig == '0A' - assert IdxBthSigDex.Ed25519_Big_Sig == '2A' - assert IdxBthSigDex.ECDSA_256k1_Big_Sig == '2C' - assert IdxBthSigDex.ECDSA_256r1_Big_Sig == '2E' - assert IdxBthSigDex.Ed448_Big_Sig == '3A' + #raw None + matter = Matter(code=code, soft=soft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # soft extra chars ignored + bigsoft = 'icprot' + matter = Matter(code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # soft bytes not str + bigsoft = b'icprot' + matter = Matter(code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # soft too small + weesoft = 'ic' + with pytest.raises(SoftMaterialError): + matter = Matter(code=code, soft=weesoft) + + # soft not B64 chars + badsoft = b'#@$%^&*!' + with pytest.raises(InvalidSoftError): + matter = Matter(code=code, soft=badsoft) + + #non empty raw and badsoft + badraw = b'abcdefg' + badsoft = b'#@$%^&*!' + with pytest.raises(InvalidSoftError): + matter = Matter(raw=badraw, code=code, soft=badsoft) + + # soft but not special code + numraw = b'\xf7\x7f' + matter = Matter(raw=numraw, code=MtrDex.Short, soft=soft) + assert matter.code == matter.hard == MtrDex.Short + assert matter.soft == '' + assert matter.raw == numraw + assert not matter.special + assert matter.composable + + # Test TBD0S '1__-' + # soft special but valid non-empty raw as part of primitive + code = MtrDex.TBD0S # sizes '1__-': Sizage(hs=4, ss=2, xs=0, fs=12, ls=0), + rs = Matter._rawSize(code) # raw size + soft = 'TG' + qb64 = '1__-TGB1dnd4' + qb2 = b'\xd7\xff\xfeL`uvwx' + raw = b'uvwx' + + assert rs == 4 + + bigsoft = 'TGIF' + extraw = b'uvwxyz' + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - # first character of code with hard size of code - assert Indexer.Hards == { - 'A': 1, 'B': 1, 'C': 1, 'D': 1, 'E': 1, 'F': 1, 'G': 1, 'H': 1, 'I': 1, - 'J': 1, 'K': 1, 'L': 1, 'M': 1, 'N': 1, 'O': 1, 'P': 1, 'Q': 1, 'R': 1, - 'S': 1, 'T': 1, 'U': 1, 'V': 1, 'W': 1, 'X': 1, 'Y': 1, 'Z': 1, - 'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1, 'f': 1, 'g': 1, 'h': 1, 'i': 1, - 'j': 1, 'k': 1, 'l': 1, 'm': 1, 'n': 1, 'o': 1, 'p': 1, 'q': 1, 'r': 1, - 's': 1, 't': 1, 'u': 1, 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1, - '0': 2, '1': 2, '2': 2, '3': 2, '4': 2, - } + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - # Codes table with sizes of code (hard) and full primitive material - assert Indexer.Sizes == { - 'A': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'B': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'C': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'D': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'E': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - 'F': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), - '0A': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '0B': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), - '2A': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2B': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2C': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2D': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2E': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '2F': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), - '3A': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '3B': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), - '0z': Xizage(hs=2, ss=2, os=0, fs=None, ls=0), - '1z': Xizage(hs=2, ss=2, os=1, fs=76, ls=1), - '4z': Xizage(hs=2, ss=6, os=3, fs=80, ls=1), - } + # Same as above but raw all zeros - assert Indexer.Sizes['A'].hs == 1 # hard size - assert Indexer.Sizes['A'].ss == 1 # soft size - assert Indexer.Sizes['A'].os == 0 # other size - assert Indexer.Sizes['A'].fs == 88 # full size - assert Indexer.Sizes['A'].ls == 0 # lead size + qb64 = '1__-TGAAAAAA' + qb2 = b'\xd7\xff\xfeL`\x00\x00\x00\x00' + raw = b'\x00\x00\x00\x00' - # verify first hs Sizes matches hs in Codes for same first char - for ckey in Indexer.Sizes.keys(): - assert Indexer.Hards[ckey[0]] == Indexer.Sizes[ckey].hs - - # verify all Codes have hs > 0 and ss > 0 and fs >= hs + ss if fs is not None - # verify os is part of ss - for val in Indexer.Sizes.values(): - assert val.hs > 0 and val.ss > 0 - assert val.os >= 0 and val.os < val.ss - if val.os: - assert val.os == val.ss // 2 - if val.fs is not None: - assert val.fs >= val.hs + val.ss - assert val.fs % 4 == 0 - - # Bizes maps bytes of sextet of decoded first character of code with hard size of code - # verify equivalents of items for Sizes and Bizes - for skey, sval in Indexer.Hards.items(): - ckey = codeB64ToB2(skey) - assert Indexer.Bards[ckey] == sval + assert rs == 4 - with pytest.raises(EmptyMaterialError): - indexer = Indexer() + bigsoft = 'TGIF' + extraw = bytearray([0] * 7) - # Test signatures - sig = (b"\x99\xd2<9$$0\x9fk\xfb\x18\xa0\x8c@r\x122.k\xb2\xc7\x1fp\x0e'm\x8f@" - b'\xaa\xa5\x8c\xc8n\x85\xc8!\xf6q\x91p\xa9\xec\xcf\x92\xaf)\xde\xca' - b'\xfc\x7f~\xd7o|\x17\x82\x1d\xd4 0 and ss > 0 and fs = hs + ss and not fs % 4 - for val in Counter.Sizes.values(): - assert val.hs > 0 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 - - # Bizes maps bytes of sextet of decoded first character of code with hard size of code - # verify equivalents of items for Sizes and Bizes - for skey, sval in Counter.Hards.items(): - ckey = codeB64ToB2(skey) - assert Counter.Bards[ckey] == sval - - with pytest.raises(EmptyMaterialError): - counter = Counter() - - # create code manually - count = 1 - qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAB' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.ControllerIdxSigs) # default count = 1 - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - # test truncates extra bytes from qb64 parameter - longqsc64 = qsc + "ABCD" - counter = Counter(qb64=longqsc64) - assert len(counter.qb64) == Counter.Sizes[counter.code].fs + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable + + # Test TBD2S '3__-' + # soft special but valid non-empty raw as part of primitive + code = MtrDex.TBD2S # sizes '2__-': Sizage(hs=4, ss=2, fs=12, ls=1), + rs = Matter._rawSize(code) # raw size + soft = 'TG' + qb64 = '3__-TGAAAHV2' # see lead byte + qb2 = b'\xdf\xff\xfeL`\x00\x00uv' + raw = b'uv' + + assert rs == 2 + + bigsoft = 'TGIF' + extraw = b'uvwxyz' + + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - # test raises ShortageError if not enough bytes in qb64 parameter - shortqsc64 = qsc[:-1] # too short - with pytest.raises(ShortageError): - counter = Counter(qb64=shortqsc64) + matter = Matter(qb2=qb2) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - # test truncates extra bytes from qb2 parameter - longqscb2 = qscb2 + bytearray([1, 2, 3, 4, 5]) # extra bytes in size - counter = Counter(qb2=longqscb2) - assert counter.qb2 == qscb2 - assert len(counter.qb64) == Counter.Sizes[counter.code].fs + matter = Matter(qb64=qb64) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - # test raises ShortageError if not enough bytes in qb2 parameter - shortqscb2 = qscb2[:-4] # too few bytes in size - with pytest.raises(ShortageError): - counter = Counter(qb2=shortqscb2) - - # test with non-zero count=5 - count = 5 - qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAF' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.ControllerIdxSigs, count=count) - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - # test with big codes index=1024 - count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) - assert qsc == '-0VAAAQA' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.BigAttachedMaterialQuadlets, count=count) - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64b=qscb) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb64=qsc) # test with str not bytes - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(qb2=qscb2) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 + # Same as above but raw all zeros - # Test ._bexfil - counter = Counter(qb64=qsc) # - code = counter.code - count = counter.count - qb2 = counter.qb2 - counter._bexfil(qb2) - assert counter.code == code - assert counter.count == count - assert counter.qb64 == qsc - assert counter.qb2 == qb2 + qb64 = '3__-TGAAAAAA' + qb2 = b'\xdf\xff\xfeL`\x00\x00\x00\x00' + raw = b'\x00\x00' - # Test ._binfil - test = counter._binfil() - assert test == qb2 + assert rs == 2 - # Test with strip - # create code manually - count = 1 - qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) - assert qsc == '-AAB' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) + bigsoft = 'TGIF' + extraw = bytearray([0] * 7) - # strip ignored if qb64 - counter = Counter(qb64=qsc, strip=True) # test with str not bytes - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - ims = bytearray(qscb) # test with qb64b - counter = Counter(qb64b=ims, strip=True) # strip - assert not ims # deleted - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - ims = bytearray(qscb2) # test with qb2 - counter = Counter(qb2=ims, strip=True) - assert not ims # deleted - assert counter.code == CtrDex.ControllerIdxSigs - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - # test with longer ims for qb64b - extra = b"ABCD" - ims = bytearray(qscb + b"ABCD") - counter = Counter(qb64b=ims, strip=True) - assert counter.qb64b == qscb - assert len(counter.qb64b) == Counter.Sizes[counter.code].fs - assert ims == extra - - # test with longer ims for qb2 - extra = bytearray([1, 2, 3, 4, 5]) - ims = bytearray(qscb2) + extra - counter = Counter(qb2=ims, strip=True) - assert counter.qb2 == qscb2 - assert len(counter.qb2) == Counter.Sizes[counter.code].fs * 3 // 4 - assert ims == extra - - # raises error if not bytearray - - ims = bytes(qscb) # test with qb64b - with pytest.raises(TypeError): - counter = Counter(qb64b=ims, strip=True) # strip - - ims = bytes(qscb2) # test with qb2 - with pytest.raises(TypeError): - counter = Counter(qb2=ims, strip=True) - - # test with big codes index=1024 - count = 1024 - qsc = CtrDex.BigAttachedMaterialQuadlets + intToB64(count, l=5) - assert qsc == '-0VAAAQA' - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - ims = bytearray(qscb) - counter = Counter(qb64b=ims, strip=True) # test with bytes not str - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - assert not ims - - ims = bytearray(qscb2) - counter = Counter(qb2=ims, strip=True) # test with qb2 - assert counter.code == CtrDex.BigAttachedMaterialQuadlets - assert counter.count == count - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - assert not ims - - # test protocol genus with CESR version - # test with big codes index=1024 - verint = 0 - version = intToB64(verint, l=3) - assert version == 'AAA' - assert verint == b64ToInt(version) - qsc = CtrDex.KERIProtocolStack + version - assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 - qscb = qsc.encode("utf-8") - qscb2 = decodeB64(qscb) - - counter = Counter(code=CtrDex.KERIProtocolStack, count=verint) - assert counter.code == CtrDex.KERIProtocolStack - assert counter.count == verint - assert counter.countToB64(l=3) == version - assert counter.countToB64() == version # default length - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - counter = Counter(code=CtrDex.KERIProtocolStack, countB64=version) - assert counter.code == CtrDex.KERIProtocolStack - assert counter.count == verint - assert counter.countToB64(l=3) == version - assert counter.countToB64() == version # default length - assert counter.qb64b == qscb - assert counter.qb64 == qsc - assert counter.qb2 == qscb2 - - assert Counter.semVerToB64("1.2.3") == "BCD" - assert Counter.semVerToB64() == "AAA" - assert Counter.semVerToB64(major=1) == "BAA" - assert Counter.semVerToB64(minor=1) == "ABA" - assert Counter.semVerToB64(patch=1) == "AAB" - assert Counter.semVerToB64(major=3, minor=4, patch=5) == "DEF" - - # test defaults for missing parts in string version - assert Counter.semVerToB64(version="1.1") == "BBA" - assert Counter.semVerToB64(version="1.") == "BAA" - assert Counter.semVerToB64(version="1") == "BAA" - assert Counter.semVerToB64(version="1.2.") == "BCA" - assert Counter.semVerToB64(version="..") == "AAA" - assert Counter.semVerToB64(version="1..3") == "BAD" - assert Counter.semVerToB64(version="4", major=1, minor=2, patch=3) == "ECD" + matter = Matter(raw=extraw, code=code, soft=bigsoft) + assert matter.code == matter.hard == code + assert matter.soft == soft + assert matter.raw == raw + assert matter.qb64 == qb64 + assert matter.qb2 == qb2 + assert matter.special + assert matter.composable - with pytest.raises(ValueError): - Counter.semVerToB64(version="64.0.1") - with pytest.raises(ValueError): - Counter.semVerToB64(version="-1.0.1") - with pytest.raises(ValueError): - Counter.semVerToB64(version="0.0.64") - with pytest.raises(ValueError): - Counter.semVerToB64(major=64) - with pytest.raises(ValueError): - Counter.semVerToB64(minor=-1) - with pytest.raises(ValueError): - Counter.semVerToB64(patch=-1) """ Done Test """ - def test_seqner(): """ - Test Seqner sequence number subclass of CryMat + Test Seqner sequence number subclass Matter """ number = Seqner() # defaults to zero assert number.raw == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' @@ -2835,6 +2193,12 @@ def test_seqner(): with pytest.raises(RawMaterialError): number = Seqner(raw=b'') + with pytest.raises(InvalidValueError): # negative + number = Seqner(sn=-1) + + with pytest.raises(ValidationError): # too big + number = Seqner(sn=(256 ** 16)) + number = Seqner(qb64b=snqb64b) assert number.raw == snraw assert number.code == MtrDex.Salt_128 @@ -2977,10 +2341,59 @@ def test_number(): """ Test Number subclass of Matter """ + + assert asdict(NumDex) == { + 'Short': 'M', + 'Long': '0H', + 'Tall': 'R', + 'Big': 'N', + 'Large': 'S', + 'Great': 'T', + 'Huge': '0A', + 'Vast': 'U' + } + + assert Number.Codes == \ + { + 'Short': 'M', + 'Long': '0H', + 'Tall': 'R', + 'Big': 'N', + 'Large': 'S', + 'Great': 'T', + 'Huge': '0A', + 'Vast': 'U' + } + + + assert Number.Names == \ + { + 'M': 'Short', + '0H': 'Long', + 'R': 'Tall', + 'N': 'Big', + 'S': 'Large', + 'T': 'Great', + '0A': 'Huge', + 'U': 'Vast' + } + + + with pytest.raises(EmptyMaterialError): + number = Number(raw=b'') # missing code + with pytest.raises(RawMaterialError): - number = Number(raw=b'') + number = Number(raw=b'', code=MtrDex.Short) # empty raw + + with pytest.raises(InvalidValueError): + number = Number(num=-1) # negative + + # when code provided does not dynamically size code + with pytest.raises(InvalidValueError): + number = Number(num=256 ** 2, code=MtrDex.Short) # wrong code for num + - number = Number() # test None defaults to zero + number = Number() # test defaults, num is None forces to zero, code dynamic assert number.code == NumDex.Short assert number.raw == b'\x00\x00' assert number.qb64 == 'MAAA' @@ -2990,6 +2403,8 @@ def test_number(): assert number.numh == '0' assert number.sn == 0 assert number.snh == '0' + assert number.huge == '0AAAAAAAAAAAAAAAAAAAAAAA' + assert len(number.huge) == 24 assert not number.positive assert number.inceptive assert hex(int.from_bytes(number.qb2, 'big')) == '0x300000' @@ -3022,6 +2437,15 @@ def test_number(): with pytest.raises(InvalidValueError): number = Number(num=" :") + # force bigger code for smaller number like for lexicographic namespace + # which must be fixed length no matter the numeric value such as sequence + # numbers in namespaces for lmdb + number = Number(num=1, code=NumDex.Huge) + assert number.qb64 == '0AAAAAAAAAAAAAAAAAAAAAAB' + assert len(number.raw) == 16 + assert NumDex.Huge == MtrDex.Salt_128 + + num = (256 ** 18 - 1) # too big to represent assert num == 22300745198530623141535718272648361505980415 numh = f"{num:x}" @@ -3057,7 +2481,6 @@ def test_number(): bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - number = Number(num=numh) # num can be hext str too assert number.code == code assert number.raw == raw @@ -3119,14 +2542,14 @@ def test_number(): bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - num = (256 ** 4 - 1) - assert num == 4294967295 + num = (256 ** 5 - 1) + assert num == 1099511627775 numh = f"{num:x}" - assert numh == 'ffffffff' - raw = b'\xff\xff\xff\xff' - code = NumDex.Long - nqb64 = '0HD_____' # '0H_____w' - nqb2 = b'\xd0p\xff\xff\xff\xff' # b'\xd0\x7f\xff\xff\xff\xf0' + assert numh == 'ffffffffff' + raw = b'\xff\xff\xff\xff\xff' + code = NumDex.Tall + nqb64 = 'RP______' # '0HD_____' # '0H_____w' + nqb2 = b'D\xff\xff\xff\xff\xff' # b'\xd0p\xff\xff\xff\xff' number = Number(num=num) assert number.code == code @@ -3193,7 +2616,7 @@ def test_number(): raw = b'\xff\xff\xff\xff\xff\xff\xff\xff' code = NumDex.Big nqb64 = 'NP__________' # 'N__________8' - nqb2 = b'4\xff\xff\xff\xff\xff\xff\xff\xff' # b'7\xff\xff\xff\xff\xff\xff\xff\xfc' + nqb2 = b'4\xff\xff\xff\xff\xff\xff\xff\xff' number = Number(num=num) assert number.code == code @@ -3255,15 +2678,14 @@ def test_number(): bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - num = (256 ** 16 - 1) - assert num == 340282366920938463463374607431768211455 + num = (256 ** 11 - 1) + assert num == 309485009821345068724781055 numh = f"{num:x}" - assert numh == 'ffffffffffffffffffffffffffffffff' - raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' - code = NumDex.Huge - nqb64 = '0AD_____________________' # '0A_____________________w' - nqb2 = b'\xd0\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' - #b'\xd0\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0' + assert numh == 'ffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Large + nqb64 = 'SP______________' # 'NP__________' # 'N__________8' + nqb2 = b'H\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' # b'4\xff\xff\xff\xff\xff\xff\xff\xff' number = Number(num=num) assert number.code == code @@ -3313,6 +2735,75 @@ def test_number(): bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + number = Number(raw=raw, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + num = (256 ** 14 - 1) + assert num == 5192296858534827628530496329220095 + numh = f"{num:x}" + assert numh == 'ffffffffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Great + nqb64 = 'TP__________________' # '0AD_____________________' + nqb2 = b'L\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + # b'\xd0\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + number = Number(numh=numh) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + number = Number(qb64=nqb64) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + number = Number(qb2=nqb2) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw number = Number(raw=raw, code=code) assert number.code == code @@ -3326,6 +2817,88 @@ def test_number(): bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + num = (256 ** 17 - 1) + assert num == 87112285931760246646623899502532662132735 + numh = f"{num:x}" + assert numh == 'ffffffffffffffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Vast + nqb64 = 'UP______________________' #'TP__________________' + nqb2 = b'P\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + # b'L\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + with pytest.raises(InvalidValueError): + number.huge # too big for huge + + number = Number(numh=numh) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + with pytest.raises(InvalidValueError): + number.huge # too big for huge + + number = Number(qb64=nqb64) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + with pytest.raises(InvalidValueError): + number.huge # too big for huge + + number = Number(qb2=nqb2) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + with pytest.raises(InvalidValueError): + number.huge # too big for huge + + + + number = Number(raw=raw, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + with pytest.raises(InvalidValueError): + number.huge # too big for huge + # tests with wrong size raw for code short num = (256 ** 2 - 1) assert num == 65535 @@ -3388,7 +2961,44 @@ def test_number(): bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw - # raw to small for code raises error + # raw too small for code raises error + raw2bad = b'\xff' + assert raw != raw2bad + assert len(raw2bad) < len(raw) + + with pytest.raises(RawMaterialError): + number = Number(raw=raw2bad, code=code) + + + # tests with wrong size raw for code large + num = (256 ** 5 - 1) + assert num == 1099511627775 + numh = f"{num:x}" + assert numh == 'ffffffffff' + raw = b'\xff\xff\xff\xff\xff' + code = NumDex.Tall + nqb64 = 'RP______' # '0HD_____' # '0H_____w' + nqb2 = b'D\xff\xff\xff\xff\xff' # b'\xd0p\xff\xff\xff\xff' + + + # raw to large for code, then truncates + raw2bad = b'\xff\xff\xff\xff\xff\xff' + assert raw != raw2bad + assert len(raw2bad) > len(raw) + + number = Number(raw=raw2bad, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + # raw too small for code raises error raw2bad = b'\xff' assert raw != raw2bad assert len(raw2bad) < len(raw) @@ -3468,6 +3078,85 @@ def test_number(): with pytest.raises(RawMaterialError): number = Number(raw=raw2bad, code=code) + + # tests with wrong size raw for code Vast + num = (256 ** 17 - 1) + assert num == 87112285931760246646623899502532662132735 + numh = f"{num:x}" + assert numh == 'ffffffffffffffffffffffffffffffffff' + raw = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + code = NumDex.Vast + nqb64 = 'UP______________________' #'TP__________________' + nqb2 = b'P\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + + + # raw to large for code, then truncates + raw2bad = b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xffxff' + assert raw != raw2bad + assert len(raw2bad) > len(raw) + + number = Number(raw=raw2bad, code=code) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + + # raw to small for code raises error + raw2bad = b'\xff' + assert raw != raw2bad + assert len(raw2bad) < len(raw) + + with pytest.raises(RawMaterialError): + number = Number(raw=raw2bad, code=code) + + + # test with negative num + num = -1 + numh = f"{num:x}" + assert numh == '-1' + code = NumDex.Short + + with pytest.raises(InvalidValueError): + number = Number(num=num) + + with pytest.raises(InvalidValueError): + number = Number(numh=numh) + + + + # test using num to initialize Number + num = 0 + numh = f"{num:x}" + assert numh == '0' + code = NumDex.Short + raw = b'\x00\x00' + nqb64 = 'MAAA' + nqb2 = b'0\x00\x00' + assert hex(int.from_bytes(nqb2, 'big')) == '0x300000' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert not number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + # test validate() + assert number.validate() == number # default inceptive = None + assert number.validate(inceptive=True) == number # inceptive = True + with pytest.raises(ValidationError): + number.validate(inceptive=False) # inceptive = False + num = 1 numh = f"{num:x}" assert numh == '1' @@ -3488,8 +3177,68 @@ def test_number(): assert number.positive bs = ceil((len(number.code) * 3) / 4) assert number.qb2[bs:] == number.raw + # test validate + assert number.validate() == number # default inceptive = None + with pytest.raises(ValidationError): + number.validate(inceptive=True) # inceptive = True + assert number.validate(inceptive=False) == number # inceptive = False + num = 65536 + numh = f"{num:x}" + assert numh == '10000' # hex + code = NumDex.Tall + raw = b'\x00\x00\x01\x00\x00' + nqb64 = 'RAAAAQAA' + nqb2 = b'D\x00\x00\x01\x00\x00' + assert hex(int.from_bytes(nqb2, 'big')) == '0x440000010000' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + # test validate + assert number.validate() == number # default inceptive = None + with pytest.raises(ValidationError): + number.validate(inceptive=True) # inceptive = True + assert number.validate(inceptive=False) == number # inceptive = False + + # too big for ordinal + num = num = (256 ** 16) + numh = f"{num:x}" + assert numh == '100000000000000000000000000000000' # hex + code = NumDex.Vast + raw =b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + nqb64 = 'UAEAAAAAAAAAAAAAAAAAAAAA' + nqb2 = b'P\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + assert hex(int.from_bytes(nqb2, 'big')) == '0x500100000000000000000000000000000000' + + number = Number(num=num) + assert number.code == code + assert number.raw == raw + assert number.qb64 == nqb64 + assert number.qb64b == nqb64.encode("utf-8") + assert number.qb2 == nqb2 + assert number.num == num + assert number.numh == numh + assert number.positive + bs = ceil((len(number.code) * 3) / 4) + assert number.qb2[bs:] == number.raw + # test validate + with pytest.raises(ValidationError): # too big + number.validate() # default inceptive = None + with pytest.raises(ValidationError): # too big + number.validate(inceptive=True) # inceptive = True + with pytest.raises(ValidationError): # too big + number.validate(inceptive=False) # inceptive = False + """ Done Test """ @@ -3596,18 +3345,691 @@ def test_dater(): assert dater.qb64b == dt1qb64b assert dater.qb2 == dt1qb2 - # datetime property and datetime math - dater1 = Dater(dts=dts1) - dater2 = Dater(dts=dts2) - dater3 = Dater(dts=helping.DTS_BASE_0) - dater4 = Dater(dts=helping.DTS_BASE_1) + # datetime property and datetime math + dater1 = Dater(dts=dts1) + dater2 = Dater(dts=dts2) + dater3 = Dater(dts=helping.DTS_BASE_0) + dater4 = Dater(dts=helping.DTS_BASE_1) + + assert dater1.datetime < dater2.datetime + assert dater4.datetime > dater3.datetime + + """ Done Test """ + +def test_tagger(): + """ + Test Tagger version primitive subclass of Matter + """ + # Test TagCodex PadTagCodex and associated Sizes to be valid specials + + with pytest.raises(EmptyMaterialError): + tagger = Tagger() # defaults + + # Tag1 + tag = 'v' + code = MtrDex.Tag1 + qb64 = '0J_v' + qb64b = qb64.encode("utf-8") + qb2 = decodeB64(qb64b) + raw = b'' + + tagger = Tagger(tag=tag) # defaults + assert tagger.code == tagger.hard == code + assert tagger.soft == tag + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + tagger = Tagger(qb2=qb2) + assert tagger.code == tagger.hard == code + assert tagger.soft == tag + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + tagger = Tagger(qb64=qb64) + assert tagger.code == tagger.hard == code + assert tagger.soft == tag + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + tagger = Tagger(qb64b=qb64b) + assert tagger.code == tagger.hard == code + assert tagger.soft == tag + assert tagger.raw == raw + assert tagger.qb64 == qb64 + assert tagger.qb2 == qb2 + assert tagger.special + assert tagger.composable + assert tagger.tag == tag + + + tags = 'abcdefghij' + alltags = dict() + for l in range(1, len(astuple(TagDex)) + 1): + tag = tags[:l] + tagger = Tagger(tag=tag) + assert tagger.tag == tag + assert len(tagger.tag) == l + assert tagger.code == astuple(TagDex)[l - 1] + alltags[l] = (tagger.tag, tagger.code) + + assert alltags == \ + { + 1: ('a', '0J'), + 2: ('ab', '0K'), + 3: ('abc', 'X'), + 4: ('abcd', '1AAF'), + 5: ('abcde', '0L'), + 6: ('abcdef', '0M'), + 7: ('abcdefg', 'Y'), + 8: ('abcdefgh', '1AAN'), + 9: ('abcdefghi', '0N'), + 10: ('abcdefghij', '0O') + } + """ Done Test """ + + +def test_ilker(): + """ + Test Ilker message type subclass of Tagger + """ + with pytest.raises(EmptyMaterialError): + ilker = Ilker() # defaults + + ilk = Ilks.rot + tag = ilk + code = MtrDex.Tag3 + soft = 'rot' + qb64 = 'Xrot' + qb64b = qb64.encode("utf-8") + qb2 = decodeB64(qb64b) + raw = b'' + + ilker = Ilker(ilk=ilk) # defaults + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(qb2=qb2) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(qb64=qb64) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(qb64b=qb64b) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + ilker = Ilker(tag=tag) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + # test error condition + with pytest.raises(InvalidSoftError): + ilker = Ilker(ilk='bad') + + # ignores code + ilker = Ilker(ilk=ilk, code=MtrDex.Tag4) + assert ilker.code == ilker.hard == code + assert ilker.soft == soft + assert ilker.raw == raw + assert ilker.qb64 == qb64 + assert ilker.qb2 == qb2 + assert ilker.special + assert ilker.composable + assert ilker.tag == tag + assert ilker.ilk == ilk + + # test error using soft and code + with pytest.raises(InvalidCodeError): + ilker = Ilker(soft='bady', code=MtrDex.Tag4) + + """End Test""" + + +def test_traitor(): + """ + Test Traitor configuration trait subclass of Tagger + """ + with pytest.raises(EmptyMaterialError): + traitor = Traitor() # defaults + + trait = TraitDex.EstOnly + tag = trait + code = MtrDex.Tag2 + soft = 'EO' + qb64 = '0KEO' + qb64b = qb64.encode("utf-8") + qb2 = decodeB64(qb64b) + raw = b'' + + traitor = Traitor(trait=trait) # defaults + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(qb2=qb2) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(qb64=qb64) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(qb64b=qb64b) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + traitor = Traitor(tag=tag) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + # test error condition + with pytest.raises(InvalidSoftError): + traitor = Traitor(trait='bad') + + # ignores code + traitor = Traitor(trait=trait, code=MtrDex.Tag4) + assert traitor.code == traitor.hard == code + assert traitor.soft == soft + assert traitor.raw == raw + assert traitor.qb64 == qb64 + assert traitor.qb2 == qb2 + assert traitor.special + assert traitor.composable + assert traitor.tag == tag + assert traitor.trait == trait + + # test error using soft and code + with pytest.raises(InvalidSoftError): + traitor = Traitor(soft='bady', code=MtrDex.Tag4) + + """End Test""" + + +def test_verser(): + """ + Test Verser version primitive subclass of Matter + """ + # Test defaults + code = MtrDex.Tag7 + soft = 'KERICAA' + tag = 'KERICAA' + qb64 = 'YKERICAA' + qb64b = qb64.encode() + qb2 = decodeB64(qb64b) + raw = b'' + versage = Versage(proto=Protocols.keri, vrsn=Vrsn_2_0, gvrsn=None) + + verser = Verser() # defaults + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.composable + assert verser.versage == versage + + # test with default equivalent values + verser = Verser(versage=versage) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.composable + assert verser.versage == versage + + verser = Verser(proto=Protocols.keri, vrsn=Vrsn_2_0) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb2=qb2) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb64=qb64) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb64b=qb64b) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + # Test with gvrsn + code = MtrDex.Tag10 + soft = 'ACDCCAACAA' + tag = 'ACDCCAACAA' + qb64 = '0OACDCCAACAA' + qb64b = qb64.encode() + qb2 = decodeB64(qb64b) + raw = b'' + versage = Versage(proto=Protocols.acdc, vrsn=Vrsn_2_0, gvrsn=Vrsn_2_0) + + verser = Verser(versage=versage) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.composable + assert verser.versage == versage + + verser = Verser(proto=Protocols.acdc, vrsn=Vrsn_2_0, gvrsn=Vrsn_2_0) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb2=qb2) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb64=qb64) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + verser = Verser(qb64b=qb64b) + assert verser.code == verser.hard == code + assert verser.soft == soft + assert verser.tag == tag + assert verser.raw == raw + assert verser.qb64 == qb64 + assert verser.qb2 == qb2 + assert verser.special + assert verser.versage == versage + + """ Done Test """ + + +def test_texter(): + """ + Test Texter variable sized text (bytes) subclass of Matter + """ + with pytest.raises(EmptyMaterialError): + texter = Texter() + + with pytest.raises(ValidationError): + texter = Texter(raw=b'Wrong code for Texter', code=MtrDex.StrB64_L0) + + + text = "" + textb = b"" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L0 + assert texter.both == '4BAA' + assert texter.raw == textb + assert texter.qb64 == '4BAA' + assert texter.text == text + + texter = Texter(text=textb) + assert texter.both == '4BAA' + assert texter.raw == b'' == textb + + texter = Texter(raw=textb) + assert texter.both == '4BAA' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '4BAA' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '4BAA' + assert texter.raw == textb + + + text = "$" + textb = b"$" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAB' + assert texter.raw == textb + assert texter.qb64 == '6BABAAAk' + assert texter.qb2 ==b'\xe8\x10\x01\x00\x00$' + assert texter.text == text + + texter = Texter(text=textb) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAB' + assert texter.raw == textb + + + + text = "@!" + textb = b"@!" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L1 + assert texter.both == '5BAB' + assert texter.raw == textb + assert texter.qb64 == '5BABAEAh' + assert texter.qb2 ==b'\xe4\x10\x01\x00@!' + assert texter.text == text + + texter = Texter(text=textb) + assert texter.both == '5BAB' + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '5BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '5BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '5BAB' + assert texter.raw == textb + + text = "^*#" + textb = b"^*#" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L0 + assert texter.both == '4BAB' + assert texter.raw == textb + assert texter.qb64 == '4BABXioj' + assert texter.qb2 == b'\xe0\x10\x01^*#' + assert texter.text == text + + texter = Texter(text=textb) + assert texter.both == '4BAB' + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '4BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '4BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '4BAB' + assert texter.raw == textb + + text = "&~?%" + textb = b"&~?%" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAC' + assert texter.raw == textb + assert texter.qb64 == '6BACAAAmfj8l' + assert texter.qb2 == b'\xe8\x10\x02\x00\x00&~?%' + assert texter.text == text + + texter = Texter(text=textb) + assert texter.both == '6BAC' + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '6BAC' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAC' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAC' + assert texter.raw == textb + + + text = "\n" # control character + textb = b"\n" + + assert len(text) == len(textb) == 1 + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAB' + assert texter.raw == textb + assert texter.qb64 == '6BABAAAK' + assert texter.qb2 ==b'\xe8\x10\x01\x00\x00\n' + assert texter.text == text + + texter = Texter(text=textb) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAB' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAB' + assert texter.raw == textb + + + text = "Did the lazy fox jumped over the big dog? But it's not its dog!\n" + textb = b"Did the lazy fox jumped over the big dog? But it's not its dog!\n" + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_L2 + assert texter.both == '6BAW' + assert texter.raw == textb + assert texter.qb64 == '6BAWAABEaWQgdGhlIGxhenkgZm94IGp1bXBlZCBvdmVyIHRoZSBiaWcgZG9nPyBCdXQgaXQncyBub3QgaXRzIGRvZyEK' + assert texter.qb2 ==(b"\xe8\x10\x16\x00\x00Did the lazy fox jumped over the big dog? But it's not " + b'its dog!\n') + assert texter.text == text + + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + + texter = Texter(text=textb) + assert texter.both == '6BAW' + assert texter.raw == textb + + texter = Texter(raw=textb) + assert texter.both == '6BAW' + assert texter.raw == textb + + texter = Texter(qb64=texter.qb64) + assert texter.both == '6BAW' + assert texter.raw == textb + + texter = Texter(qb2=texter.qb2) + assert texter.both == '6BAW' + assert texter.raw == textb + + + + text = "a" * ((64 ** 2) * 3) # big variable size + textb = text.encode("utf-8") - assert dater1.datetime < dater2.datetime - assert dater4.datetime > dater3.datetime + assert len(text) // 3 > (64 ** 2 - 1) + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_Big_L0 + assert texter.both == '7AABABAA' + assert texter.raw == textb + assert len(texter.qb64) == 16392 + assert len(texter.qb2) == 12294 + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + assert texter.text == text + + text = "b" * ((64 ** 2 ) * 3 + 1) # big variable size + textb = text.encode("utf-8") + + assert len(text) // 3 > (64 ** 2 - 1) + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_Big_L2 + assert texter.both == '9AABABAB' + assert texter.raw == textb + assert len(texter.qb64) == 16396 + assert len(texter.qb2) == 12297 + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + assert texter.text == text + + text = "c" * ((64 ** 2 ) * 3 + 2) # big variable size + textb = text.encode("utf-8") + + assert len(text) // 3 > (64 ** 2 - 1) + + texter = Texter(text=text) + assert texter.code == MtrDex.Bytes_Big_L1 + assert texter.both == '8AABABAB' + assert texter.raw == textb + assert len(texter.qb64) == 16396 + assert len(texter.qb2) == 12297 + assert len(texter.qb64) * 3 / 4 == len(texter.qb2) + assert texter.text == text + + text = "c" * ((64 ** 4) * 3) # excessive variable size + with pytest.raises(InvalidVarRawSizeError): + texter = Texter(text=text) """ Done Test """ + def test_bexter(): """ Test Bexter variable sized Base64 text subclass of Matter @@ -3615,6 +4037,9 @@ def test_bexter(): with pytest.raises(EmptyMaterialError): bexter = Bexter() + with pytest.raises(ValidationError): + bexter = Bexter(raw=b'Wrong_code_for_Bexter', code=MtrDex.Bytes_L0) + bext = "@!" with pytest.raises(ValueError): bexter = Bexter(bext=bext) @@ -3846,6 +4271,194 @@ def test_pather(): """ Done Test """ +def test_labeler(): + """ + Test Labeler subclass of Matter + """ + with pytest.raises(EmptyMaterialError): + labeler = Labeler() # defaults + + # test taggable label + label = 'z' + raw = b'' + code = LabelDex.Tag1 + qb64 = '0J_z' + qb2 = decodeB64(qb64) + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == label + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code, soft=label) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + + # Test all sizes taggable labels + labels = ('A', 'AB', 'ABC', 'ABCD', 'ABCDE', 'ABCDEF', 'ABCDEFG', 'ABCDEFGH', + 'ABCDEFGHI', 'ABCDEFGHIJ') + + raw = b'' + for i, label in enumerate(labels): + code = astuple(LabelDex)[i] + xs = Matter._xtraSize(code) + qb64 = code + ('_' * xs) + label + qb2 = decodeB64(qb64) + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == label + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code, soft=label) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + + # test bextable labels + label = 'zyxwvutsrqponm' + code = LabelDex.StrB64_L1 + qb64 = '5AAEAAzyxwvutsrqponm' + qb2 = decodeB64(qb64) + raw = qb2[4:] # skip 3 for code and 1 for lead pad + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + rs = (len(label) + len(label) % 4) // 4 + assert labeler.soft == intToB64(rs, 2) == 'AE' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code, soft=label) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + # test textable labels + # fixed size short + label = '@' + code = LabelDex.Label1 + raw = label.encode() + qb64 = 'VABA' + qb2 = decodeB64(qb64) # b'T\x00@' + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == '' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + label = '!$' + code = LabelDex.Label2 + raw = label.encode() + qb64 = 'WCEk' + qb2 = decodeB64(qb64) # b'X!$' + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == '' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + + # variable sized + label = '#yxwvutsrqponm' + code = LabelDex.Bytes_L1 + raw = label.encode() + qb64 = '5BAFACN5eHd2dXRzcnFwb25t' + qb2 = decodeB64(qb64) + + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == 'AF' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + # test base64 that starts with 'A' get encoded as textable, is not bextable + label = 'Ayxwvutsrqponm' + code = LabelDex.Bytes_L1 + raw = label.encode() + qb64 = '5BAFAEF5eHd2dXRzcnFwb25t' + qb2 = decodeB64(qb64) + + + labeler = Labeler(label=label) + assert labeler.label == label + assert labeler.code == code + assert labeler.soft == 'AF' + assert labeler.raw == raw + assert labeler.qb64 == qb64 + assert labeler.qb2 == qb2 + + labeler = Labeler(raw=raw, code=code) + assert labeler.label == label + + labeler = Labeler(qb64=qb64) + assert labeler.label == label + + labeler = Labeler(qb2=qb2) + assert labeler.label == label + + """ Done Test """ + + def test_verfer(): """ Test the support functionality for verifier subclass of crymat @@ -3980,7 +4593,7 @@ def test_verfer(): def test_cigar(): """ - Test Cigar subclass of CryMat + Test Cigar subclass of Matter """ with pytest.raises(EmptyMaterialError): cigar = Cigar() @@ -4003,695 +4616,15 @@ def test_cigar(): """ Done Test """ -def test_signer(): - """ - Test the support functionality for signer subclass of crymat - """ - signer = Signer() # defaults provide Ed25519 signer Ed25519 verfer - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - # create something to sign and verify - ser = b'abcdefghijklmnopqrstuvwxyz0123456789' - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.Ed25519_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - index = 0 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.Ed25519_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - assert cigar.raw == siger.raw - - with pytest.raises(ValueError): # use invalid code not SEED type code - signer = Signer(code=MtrDex.Ed25519N) - - # Non transferable defaults - signer = Signer(transferable=False) # Ed25519N verifier - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519N - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.Ed25519_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - siger = signer.sign(ser, index=0) - assert siger.code == IdrDex.Ed25519_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - - # non default seed - seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - signer = Signer(raw=seed, code=MtrDex.Ed25519_Seed) - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.Ed25519_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - index = 1 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.Ed25519_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - assert cigar.raw == siger.raw - - # different both so Big - ondex = 3 - siger = signer.sign(ser, index=index, ondex=ondex) - assert siger.code == IdrDex.Ed25519_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == ondex - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # same but Big - index = 67 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.Ed25519_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # different both so Big - ondex = 67 - siger = signer.sign(ser, index=index, ondex=ondex) - assert siger.code == IdrDex.Ed25519_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == ondex - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # current only - index = 4 - siger = signer.sign(ser, index=index, only=True) - assert siger.code == IdrDex.Ed25519_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # ignores ondex if only - siger = signer.sign(ser, index=index, only=True, ondex=index+2) - assert siger.code == IdrDex.Ed25519_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # big current only - index = 65 - siger = signer.sign(ser, index=index, only=True) - assert siger.code == IdrDex.Ed25519_Big_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # ignores ondex if only - siger = signer.sign(ser, index=index, only=True, ondex=index+2) - assert siger.code == IdrDex.Ed25519_Big_Crt_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == None - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - with pytest.raises(ValueError): # use invalid code not SEED type code - signer = Signer(raw=seed, code=MtrDex.Ed25519N) - - # Test Secp256r1, default seed - signer = Signer(code=MtrDex.ECDSA_256r1_Seed) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256r1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result is True - - # Test non-default seed - seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - # Test hardcoded seed - seed = (b'\x9f{\xa8\xa7\xa8C9\x96&\xfa\xb1\x99\xeb\xaa \xc4\x1bG\x11\xc4\xaeSAR\xc9\xbd\x04\x9d\x85)~\x93') - signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == "QJ97qKeoQzmWJvqxmeuqIMQbRxHErlNBUsm9BJ2FKX6T" - assert signer.verfer.qb64 == "1AAJA3cK_P2CDlh-_EMFPvyqTPI1POkw-dr14DANx5JEXDCZ" - - # Test vectors from CERSide - seed = (b'\x35\x86\xc9\xa0\x4d\x33\x67\x85\xd5\xe4\x6a\xda\x62\xf0\x54\xc5\xa5\xf4\x32\x3f\x46\xcb\x92\x23\x07' - b'\xe0\xe2\x79\xb7\xe5\xf5\x0a') - verkey = (b"\x03\x16\x99\xbc\xa0\x51\x8f\xa6\x6c\xb3\x5d\x6b\x0a\x92\xf6\x84\x96\x28\x7b\xb6\x64\xe8\xe8\x57\x69" - b"\x15\xb8\xea\x9a\x02\x06\x2a\xff") - sig = (b'\x8c\xfa\xb4\x40\x01\xd2\xab\x4a\xbc\xc5\x96\x8b\xa2\x65\x76\xcd\x51\x9d\x3b\x40\xc3\x35\x21\x73\x9a\x1b' - b'\xe8\x2f\xe1\x30\x28\xe1\x07\x90\x08\xa6\x42\xd7\x3f\x36\x8c\x96\x32\xff\x01\x64\x03\x18\x08\x85\xb8\xa4' - b'\x97\x76\xbe\x9c\xe4\xd7\xc5\xe7\x05\xda\x51\x23') - - signerqb64 = "QDWGyaBNM2eF1eRq2mLwVMWl9DI_RsuSIwfg4nm35fUK" - verferqb64 = "1AAJAxaZvKBRj6Zss11rCpL2hJYoe7Zk6OhXaRW46poCBir_" - cigarqb64 = "0ICM-rRAAdKrSrzFlouiZXbNUZ07QMM1IXOaG-gv4TAo4QeQCKZC1z82jJYy_wFkAxgIhbikl3a-nOTXxecF2lEj" - - ser = b'abc' - signer = Signer(raw=seed, code=MtrDex.ECDSA_256r1_Seed) - cigar = signer.sign(ser) - assert signer.code == MtrDex.ECDSA_256r1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.qb64 == signerqb64 - - assert signer.verfer.code == MtrDex.ECDSA_256r1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.verfer.raw == verkey - assert signer.verfer.qb64 == verferqb64 - - assert cigar.code == MtrDex.ECDSA_256r1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - assert signer.verfer.verify(cigar.raw, ser) - assert signer.verfer.verify(sig, ser) - - cigar = Cigar(raw=sig, code=MtrDex.ECDSA_256r1_Sig) - assert cigar.qb64 == cigarqb64 - - - # Test Secp256k1, default seed - signer = Signer(code=MtrDex.ECDSA_256k1_Seed) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - # create something to sign and verify - ser = b'abcdefghijklmnopqrstuvwxyz0123456789' - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result is True - - index = 0 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.ECDSA_256k1_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - # Non transferable - signer = Signer(code=MtrDex.ECDSA_256k1_Seed, transferable=False) # ECDSA_256k1N verifier - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.ECDSA_256k1N - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - siger = signer.sign(ser, index=0) - assert siger.code == IdrDex.ECDSA_256k1_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - # Test non-default seed - seed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - - cigar = signer.sign(ser) - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - result = signer.verfer.verify(cigar.raw, ser) - assert result == True - - index = 1 - siger = signer.sign(ser, index=index) - assert siger.code == IdrDex.ECDSA_256k1_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == index - result = signer.verfer.verify(siger.raw, ser) - assert result == True - result = signer.verfer.verify(siger.raw, ser + b'ABCDEFG') - assert result == False - - # different both so Big - ondex = 3 - siger = signer.sign(ser, index=index, ondex=ondex) - assert siger.code == IdrDex.ECDSA_256k1_Big_Sig - assert len(siger.raw) == Indexer._rawSize(siger.code) - assert siger.index == index - assert siger.ondex == ondex - result = signer.verfer.verify(siger.raw, ser) - assert result == True - - # Test hardcoded seed from CERSide - seed = (b'\x9f{\xa8\xa7\xa8C9\x96&\xfa\xb1\x99\xeb\xaa \xc4\x1bG\x11\xc4\xaeSAR\xc9\xbd\x04\x9d\x85)~\x93') - signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == "JJ97qKeoQzmWJvqxmeuqIMQbRxHErlNBUsm9BJ2FKX6T" - assert signer.verfer.qb64 == "1AABAg299p5IMvuw71HW_TlbzGq5cVOQ7bRbeDuhheF-DPYk" - - # Test vectors from CERSide - seed = (b'\x7f\x98\x0a\x3b\xe4\x45\xd7\x8c\xc9\x79\xa1\xee\x26\x20\x9c\x17\x71\x16\xab\xa6\xd6\xf1\x6a\x01\xe7\xb3\xce\xfe\xe2\x6c\x06\x08') - verkey = (b"\x02\xdb\x98\x33\x85\xa8\x0e\xbb\x7c\x15\x5d\xdd\xc6\x47\x6a\x24\x07\x9a\x7c\x96\x5f\x05\x0f\x62\xde\x2d\x47\x56\x9b\x54\x29\x16\x79") - sig = (b'\x5f\x80\xc0\x5a\xe4\x71\x32\x5d\xf7\xcb\xdb\x1b\xc2\xf4\x11\xc3\x05\xaf\xf4\xbe\x3b\x7e\xac\x3e\x8c\x15' - b'\x3a\x9f\xa5\x0a\x3d\x69\x75\x45\x93\x34\xc8\x96\x2b\xfe\x79\x8d\xd1\x4e\x9c\x1f\x6c\xa7\xc8\x12\xd6' - b'\x7a\x6c\xc5\x74\x9f\xef\x8d\xa7\x25\xa2\x95\x47\xcc') - - signerqb64 = "JH-YCjvkRdeMyXmh7iYgnBdxFqum1vFqAeezzv7ibAYI" - verferqb64 = "1AABAtuYM4WoDrt8FV3dxkdqJAeafJZfBQ9i3i1HVptUKRZ5" - cigarqb64 = "0CBfgMBa5HEyXffL2xvC9BHDBa_0vjt-rD6MFTqfpQo9aXVFkzTIliv-eY3RTpwfbKfIEtZ6bMV0n--NpyWilUfM" - - ser = b'abc' - signer = Signer(raw=seed, code=MtrDex.ECDSA_256k1_Seed) - cigar = signer.sign(ser) - assert signer.code == MtrDex.ECDSA_256k1_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.raw == seed - assert signer.qb64 == signerqb64 - - assert signer.verfer.code == MtrDex.ECDSA_256k1 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.verfer.raw == verkey - assert signer.verfer.qb64 == verferqb64 - - assert cigar.code == MtrDex.ECDSA_256k1_Sig - assert len(cigar.raw) == Matter._rawSize(cigar.code) - assert signer.verfer.verify(cigar.raw, ser) - assert signer.verfer.verify(sig, ser) - - cigar = Cigar(raw=sig, code=MtrDex.ECDSA_256k1_Sig) - assert cigar.qb64 == cigarqb64 - - - # test with only and ondex parameters - - """ Done Test """ - -def test_cipher(): - """ - Test Cipher subclass of Matter - """ - # conclusion never use box_seed_keypair always use sign_seed_keypair and - # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey - # is always the same. - - assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 - - # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' - b'\xde\x06\xc0+') - seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b - assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' - #b'AGDswxA8qdkb646JFZWUflm_OKUeF41iG2gTw3N4GwCs' - - # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' - saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b - assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' - - # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) - cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) - - with pytest.raises(EmptyMaterialError): - cipher = Cipher() - - raw = pysodium.crypto_box_seal(seedqb64b, pubkey) # uses nonce so different everytime - cipher = Cipher(raw=raw) - assert cipher.code == MtrDex.X25519_Cipher_Seed - uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == seedqb64b - - # test .decrypt method needs qb64 - prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b - assert cipher.decrypt(prikey=prikeyqb64).qb64b == seedqb64b - - cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b - assert cipher.decrypt(seed=cryptseedqb64).qb64b == seedqb64b - - raw = pysodium.crypto_box_seal(saltqb64b, pubkey) # uses nonce so different everytime - cipher = Cipher(raw=raw) - assert cipher.code == MtrDex.X25519_Cipher_Salt - uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) - assert uncb == saltqb64b - - # test .decrypt method needs qb64 - prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b - assert cipher.decrypt(prikey=prikeyqb64).qb64b == saltqb64b - - cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b - assert cipher.decrypt(seed=cryptseedqb64).qb64b == saltqb64b - - with pytest.raises(ValueError): # bad code - cipher = Cipher(raw=raw, code=MtrDex.Ed25519N) - """ Done Test """ - - -def test_encrypter(): - """ - Test Encrypter subclass of Matter - """ - # conclusion never use box_seed_keypair always use sign_seed_keypair and - # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey - # is always the same. - - assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 - - # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' - b'\xde\x06\xc0+') - seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b - assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' - - # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' - saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b - assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' - - # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) - cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) - verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) - - with pytest.raises(EmptyMaterialError): - encrypter = Encrypter() - - encrypter = Encrypter(raw=pubkey) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - assert encrypter.verifySeed(seed=cryptsigner.qb64) - - cipher = encrypter.encrypt(ser=seedqb64b) - assert cipher.code == MtrDex.X25519_Cipher_Seed - uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) - assert uncb == seedqb64b - - cipher = encrypter.encrypt(ser=saltqb64b) - assert cipher.code == MtrDex.X25519_Cipher_Salt - uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) - assert uncb == saltqb64b - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) - - encrypter = Encrypter(verkey=verfer.qb64) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - - encrypter = Encrypter(verkey=verfer.qb64b) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - - # user Prefixer to generate original verkey - prefixer = Prefixer(qb64=verfer.qb64) - encrypter = Encrypter(verkey=prefixer.qb64b) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - """ Done Test """ - - -def test_decrypter(): - """ - Test Decrypter subclass of Matter - """ - # conclusion never use box_seed_keypair always use sign_seed_keypair and - # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey - # is always the same. - - assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 - - # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) - seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' - b'\xde\x06\xc0+') - signer = Signer(raw=seed, code=MtrDex.Ed25519_Seed) - assert signer.verfer.code == MtrDex.Ed25519 - assert signer.verfer.transferable # default - seedqb64b = signer.qb64b - assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' - # also works for Matter - assert seedqb64b == Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b - - # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - raw = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' - salter = Salter(raw=raw, code=MtrDex.Salt_128) - assert salter.code == MtrDex.Salt_128 - saltqb64b = salter.qb64b - assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' - # also works for Matter - assert saltqb64b == Matter(raw=raw, code=MtrDex.Salt_128).qb64b # - - # cryptseed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) - cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) - verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw - pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) - prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) - - with pytest.raises(EmptyMaterialError): - decrypter = Decrypter() - - # create encrypter - encrypter = Encrypter(raw=pubkey) - assert encrypter.code == MtrDex.X25519 - assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' - assert encrypter.raw == pubkey - - # create cipher of seed - seedcipher = encrypter.encrypt(ser=seedqb64b) - assert seedcipher.code == MtrDex.X25519_Cipher_Seed - # each encryption uses a nonce so not a stable representation for testing - - # create decrypter from prikey - decrypter = Decrypter(raw=prikey) - assert decrypter.code == MtrDex.X25519_Private - assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' - assert decrypter.raw == prikey - - # decrypt seed cipher using ser - designer = decrypter.decrypt(ser=seedcipher.qb64b, transferable=signer.verfer.transferable) - assert designer.qb64b == seedqb64b - assert designer.code == MtrDex.Ed25519_Seed - assert designer.verfer.code == MtrDex.Ed25519 - assert signer.verfer.transferable - - # decrypt seed cipher using cipher - designer = decrypter.decrypt(cipher=seedcipher, transferable=signer.verfer.transferable) - assert designer.qb64b == seedqb64b - assert designer.code == MtrDex.Ed25519_Seed - assert designer.verfer.code == MtrDex.Ed25519 - assert signer.verfer.transferable - - # create cipher of salt - saltcipher = encrypter.encrypt(ser=saltqb64b) - assert saltcipher.code == MtrDex.X25519_Cipher_Salt - # each encryption uses a nonce so not a stable representation for testing - - # decrypt salt cipher using ser - desalter = decrypter.decrypt(ser=saltcipher.qb64b) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - # decrypt salt cipher using cipher - desalter = decrypter.decrypt(cipher=saltcipher) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - # use previously stored fully qualified seed cipher with different nonce - # get from seedcipher above - cipherseed = ('PM9jOGWNYfjM_oLXJNaQ8UlFSAV5ACjsUY7J16xfzrlpc9Ve3A5WYrZ4o_' - 'NHtP5lhp78Usspl9fyFdnCdItNd5JyqZ6dt8SXOt6TOqOCs-gy0obrwFkPPqBvVkEw') - designer = decrypter.decrypt(ser=cipherseed, transferable=signer.verfer.transferable) - assert designer.qb64b == seedqb64b - assert designer.code == MtrDex.Ed25519_Seed - assert designer.verfer.code == MtrDex.Ed25519 - - # use previously stored fully qualified salt cipher with different nonce - # get from saltcipher above - ciphersalt = ('1AAHjlR2QR9J5Et67Wy-ZaVdTryN6T6ohg44r73GLRPnHw-5S3ABFkhWy' - 'IwLOI6TXUB_5CT13S8JvknxLxBaF8ANPK9FSOPD8tYu') - desalter = decrypter.decrypt(ser=ciphersalt) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - # Create new decrypter but use seed parameter to init prikey - decrypter = Decrypter(seed=cryptsigner.qb64b) - assert decrypter.code == MtrDex.X25519_Private - assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' - assert decrypter.raw == prikey - - # decrypt ciphersalt - desalter = decrypter.decrypt(ser=saltcipher.qb64b) - assert desalter.qb64b == saltqb64b - assert desalter.code == MtrDex.Salt_128 - - """ Done Test """ - - -def test_salter(): - """ - Test the support functionality for salter subclass of crymat - """ - salter = Salter() # defaults to CryTwoDex.Salt_128 - assert salter.code == MtrDex.Salt_128 - assert len(salter.raw) == Matter._rawSize(salter.code) == 16 - - raw = b'0123456789abcdef' - salter = Salter(raw=raw) - assert salter.raw == raw - assert salter.qb64 == '0AAwMTIzNDU2Nzg5YWJjZGVm' #'0ACDEyMzQ1Njc4OWFiY2RlZg' - - signer = salter.signer(path="01", temp=True) # defaults to Ed25519 - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == 'AMPsqBZxWdtYpBhrWnKYitwFa77s902Q-nX3sPTzqs0R' - #'Aw-yoFnFZ21ikGGtacpiK3AVrvuz3TZD6dfew9POqzRE' - assert signer.verfer.qb64 == 'DFYFwZJOMNy3FknECL8tUaQZRBUyQ9xCv6F8ckG-UCrC' # - # 'DVgXBkk4w3LcWScQIvy1RpBlEFTJD3EK_oXxyQb5QKsI' - - signer = salter.signer(path="01") # defaults to Ed25519 temp = False level="low" - assert signer.code == MtrDex.Ed25519_Seed - assert len(signer.raw) == Matter._rawSize(signer.code) - assert signer.verfer.code == MtrDex.Ed25519 - assert len(signer.verfer.raw) == Matter._rawSize(signer.verfer.code) - assert signer.qb64 == 'AEkqQiNTexWB9fTLpgJp_lXW63tFlT-Y0_mgQww4o-dC' - # 'ASSpCI1N7FYH19MumAmn-Vdbre0WVP5jT-aBDDDij50I' - assert signer.verfer.qb64 == 'DPJGyH9H1M_SUSf18RzX8OqdyhxEyZJpKm5Em0PnpsWd' - #'D8kbIf0fUz9JRJ_XxHNfw6p3KHETJkmkqbkSbQ-emxZ0' - - salter = Salter(qb64='0AAwMTIzNDU2Nzg5YWJjZGVm') - assert salter.raw == raw - assert salter.qb64 == '0AAwMTIzNDU2Nzg5YWJjZGVm' - - with pytest.raises(ShortageError): - salter = Salter(qb64='') - - salter = Salter(raw=raw) - assert salter.stretch(temp=True) == b'\xd4@\xeb\xa6x\x86\xdf\x93\xd6C\xdc\xb8\xa6\x9b\x02\xafh\xc1m(L\xd6\xf6\x86YU>$[\xf9\xef\xc0' - assert salter.stretch(tier=Tiers.low) == b'\xf8e\x80\xbaX\x08\xb9\xba\xc6\x1e\x84\r\x1d\xac\xa7\\\x82Wc@`\x13\xfd\x024t\x8ct\xd3\x01\x19\xe9' - assert salter.stretch(tier=Tiers.med) == b',\xf3\x8c\xbb\xe9)\nSQ\xec\xad\x8c9?\xaf\xb8\xb0\xb3\xcdB\xda\xd8\xb6\xf7\r\xf6D}Z\xb9Y\x16' - assert salter.stretch(tier=Tiers.high) == b'(\xcd\xc4\xb85\xcd\xe8:\xfc\x00\x8b\xfd\xa6\tj.y\x98\x0b\x04\x1c\xe3hBc!I\xe49K\x16-' - - """ Done Test """ - - -def test_generatesigners(): - """ - Test the support function genSigners - - """ - signers = generateSigners(count=2, transferable=False) - assert len(signers) == 2 - for signer in signers: - assert signer.verfer.code == MtrDex.Ed25519N - - # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) - salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - assert len(salt) == 16 - signers = generateSigners(salt=salt, count=4) # default is transferable - assert len(signers) == 4 - for signer in signers: - assert signer.code == MtrDex.Ed25519_Seed - assert signer.verfer.code == MtrDex.Ed25519 - - sigkeys = [signer.qb64 for signer in signers] - assert sigkeys == ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', - 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', - 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', - 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP'] - - secrets = generatePrivates(salt=salt, count=4) - assert secrets == sigkeys - - """ End Test """ - def test_diger(): """ Test the support functionality for Diger subclass of CryMat """ + # Ensure keyspace of Diger.Digests is same as codes in DigDex + assert set(coring.DigDex) == set(Diger.Digests.keys()) + + with pytest.raises(EmptyMaterialError): diger = Diger() @@ -4699,11 +4632,9 @@ def test_diger(): ser = b'abcdefghijklmnopqrstuvwxyz0123456789' dig = blake3.blake3(ser).digest() - with pytest.raises(coring.InvalidValueError): + with pytest.raises(kering.InvalidCodeError): diger = Diger(raw=dig, code=MtrDex.Ed25519) - with pytest.raises(coring.InvalidValueError): - diger = Diger(ser=ser, code=MtrDex.Ed25519) diger = Diger(raw=dig) # defaults provide Blake3_256 digester assert diger.code == MtrDex.Blake3_256 @@ -4842,8 +4773,8 @@ def test_prefixer(): with pytest.raises(EmptyMaterialError): prefixer = Prefixer(raw=verkey, code='') - with pytest.raises(ValueError): - prefixer = Prefixer(raw=verkey, code=MtrDex.SHA2_256) + #with pytest.raises(InvalidCodeError): + #prefixer = Prefixer(raw=verkey, code=MtrDex.SHA2_256) # test creation given raw and code no derivation prefixer = Prefixer(raw=verkey, code=MtrDex.Ed25519N) # default code is None @@ -4851,491 +4782,32 @@ def test_prefixer(): assert len(prefixer.raw) == Matter._rawSize(prefixer.code) assert len(prefixer.qb64) == Matter.Sizes[prefixer.code].fs - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="", - n=[], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="1", - n=["ABCD"], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == False - assert prefixer.verify(ked=ked, prefixed=True) == False prefixer = Prefixer(raw=verkey, code=MtrDex.Ed25519) # defaults provide Ed25519N prefixer assert prefixer.code == MtrDex.Ed25519 assert len(prefixer.raw) == Matter._rawSize(prefixer.code) assert len(prefixer.qb64) == Matter.Sizes[prefixer.code].fs - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="1", - n=["ABCD"], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False + verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) prefixer = Prefixer(raw=verfer.raw, code=MtrDex.Ed25519N) assert prefixer.code == MtrDex.Ed25519N - assert prefixer.verify(ked=ked) == False - assert prefixer.verify(ked=ked, prefixed=True) == False - - # Test basic derivation from ked - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[verfer.qb64], # list of qb64 - nt="", - n=0, # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Ed25519) - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - badked = dict(ked) - del badked["i"] - with pytest.raises(EmptyMaterialError): # no pre - prefixer = Prefixer(ked=badked) - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - with pytest.raises(DerivationError): # verfer code not match pre code - prefixer = Prefixer(ked=badked) - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - with pytest.raises(DerivationError): - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519N) # verfer code not match code - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519N) # verfer code match code but not pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == False - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519N) # verfer code match code and pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked) # verfer code match pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - del badked["i"] - with pytest.raises(EmptyMaterialError): # missing pre - prefixer = Prefixer(ked=badked) - - verfer = Verfer(raw=verkey, code=MtrDex.Ed25519N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - with pytest.raises(ShortageError): # empty pre - prefixer = Prefixer(ked=badked) - - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["n"] = "ABCD" - with pytest.raises(DerivationError): # wrong code for transferable - prefixer = Prefixer(ked=badked, code=MtrDex.Ed25519) - - # Test digest derivation from inception ked - vs = versify(version=Version, kind=Serials.json, size=0) - sn = 0 - ilk = Ilks.icp - sith = "1" - keys = [Prefixer(raw=verkey, code=MtrDex.Ed25519).qb64] - nxt = "" - toad = 0 - wits = [] - cnfg = [] - - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=0, - n=[], - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EEZn82xRQYFjfkPJ5ECrDNHJ6xSt_hjxybbt_WMpinEF' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # test with next digs - ndigs = [Diger(ser=nxtfer.qb64b).qb64] - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EHB9-i6jOH6DbK_40vlGF0X78Mg__c3MSzu9AE9ZRrsC' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - - salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - #secrets = generateSecrets(salt=salt, count=8) - - # test with fractionally weighted sith - secrets = ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', - 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', - 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', - 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP', - 'ANfkMQ5LKPfjEdQPK2c_zWsOn4GgLWsnWvIa25EVVbtR', - 'ACrmDHtPQjnM8H9pyKA-QBNdfZ-xixTlRZTS8WXCrrMH', - 'AMRXyU3ErhBNdRSDX1zKlrbZGRp1GfCmkRIa58gF07I8', - 'AC6vsNVCpHa6acGcxk7c-D1mBHlptPrAx8zr-bKvesSW'] - - # create signers from secrets - signers = [Signer(qb64=secret) for secret in secrets] # faster - assert [siger.qb64 for siger in signers] == secrets - # each signer has verfer for keys - - # Test with sith with one clause - keys = [signers[0].verfer.qb64, signers[1].verfer.qb64, signers[2].verfer.qb64] - sith = [["1/2", "1/2", "1"]] - ndigs = [Diger(ser=signers[3].verfer.qb64b).qb64] # default limen/sith - - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer1 = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer1.qb64 == 'EOnpRzJpF1LNdCXl7aQ76BxF7qT94PChM7WGKARhZeKj' - assert prefixer1.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # now test with different sith but same weights in two clauses - sith = [["1/2", "1/2"], ["1"]] - - ked = dict(v=vs, # version string - t=ilk, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - - prefixer2 = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer2.qb64 == 'ECBv9o83MnNYRTdXhwTeR5zgwt8jTr5NIuJ8P00BKySW' - assert prefixer2.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - assert prefixer2.qb64 != prefixer1.qb64 # semantic diff -> syntactic diff - - sith = "1" - seal = dict(i='EBfPkd-A2CQfJmfpmtc1V-yuleSeCcyWBIrTAygUgQ_T', - s='2', - t=Ilks.ixn, - d='EB0_D51cTh_q6uOQ-byFiv5oNXZ-cxdqCqBAa4JmBLtb') - - ked = dict(v=vs, # version string - t=Ilks.dip, - d="", # SAID - i="", # qb64 prefix - s="{:x}".format(sn), # hex string no leading zeros lowercase - kt=sith, # hex string no leading zeros lowercase - k=keys, # list of qb64 - nt=1, - n=ndigs, # hash qual Base64 - bt="{:x}".format(toad), # hex string no leading zeros lowercase - b=wits, # list of qb64 may be empty - c=cnfg, # list of config ordered mappings may be empty - a=[seal], # list of seal dicts - di='EBfPkd-A2CQfJmfpmtc1V-yuleSeCcyWBIrTAygUgQ_T', - ) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EEGithHj9A85F9hz1fxlF80U7wvpFoAPj6U4q4YWMehp' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # test with allows - with pytest.raises(ValueError): - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256, - allows=[MtrDex.Ed25519N, MtrDex.Ed25519]) - - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256, - allows=[MtrDex.Blake3_256, MtrDex.Ed25519]) - assert prefixer.qb64 == 'EEGithHj9A85F9hz1fxlF80U7wvpFoAPj6U4q4YWMehp' - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - # Secp256r1 - - preN = '1AAIA-KzxCX8SZSl-fpU3vc3z_MBuH06YShJFuiMdAmo37TM' - # 'BrHLayDN-mXKv62DAjFLX1_Y5yEUe0vA9YPe_ihiKYHE' - pre = '1AAJA-KzxCX8SZSl-fpU3vc3z_MBuH06YShJFuiMdAmo37TM' - - # sigkey = ec.generate_private_key(ec.SECP256R1()) - # verkey = sigkey.public_key().public_bytes(encoding=Encoding.X962, format=PublicFormat.CompressedPoint) - verkey = b'\x03\xe2\xb3\xc4%\xfcI\x94\xa5\xf9\xfaT\xde\xf77\xcf\xf3\x01\xb8}:a(I\x16\xe8\x8ct\t\xa8\xdf\xb4\xcc' - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - assert verfer.qb64 == '1AAJA-KzxCX8SZSl-fpU3vc3z_MBuH06YShJFuiMdAmo37TM' - - nxtkeyqb64 = [coring.Diger(ser=verfer.qb64b).qb64] # dfault sith is 1 - assert nxtkeyqb64 == ['EPrVv1ppjxrtV48cS9Tm49n5xojMlZfhEzExg6Ye_ORN'] - prefixer = Prefixer(raw=verkey, code=MtrDex.ECDSA_256r1) # default code is None - assert prefixer.code == MtrDex.ECDSA_256r1 - assert len(prefixer.raw) == Matter._rawSize(prefixer.code) - assert len(prefixer.qb64) == Matter.Sizes[prefixer.code].fs - - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[prefixer.qb64], # list of qb64 - nt="1", - n=["ABCD"], # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - prefixer = Prefixer(raw=verfer.raw, code=MtrDex.ECDSA_256r1N) - assert prefixer.code == MtrDex.ECDSA_256r1N - assert prefixer.verify(ked=ked) == False - assert prefixer.verify(ked=ked, prefixed=True) == False - - # Test basic derivation from ked - ked = dict(v="", # version string - t="icp", - d="", # qb64 SAID - i="", # qb64 prefix - s="0", # hex string no leading zeros lowercase - kt=1, - k=[verfer.qb64], # list of qb64 - nt="", - n=0, # hash qual Base64 - bt=0, - b=[], # list of qb64 may be empty - c=[], # list of config ordered mappings may be empty - a=[], # list of seal dicts - ) - prefixer = Prefixer(ked=ked, code=MtrDex.ECDSA_256r1) - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=ked) == True - assert prefixer.verify(ked=ked, prefixed=True) == False - - badked = dict(ked) - del badked["i"] - with pytest.raises(EmptyMaterialError): # no pre - prefixer = Prefixer(ked=badked) - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - with pytest.raises(DerivationError): # verfer code not match pre code - prefixer = Prefixer(ked=badked) - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - with pytest.raises(DerivationError): - prefixer = Prefixer(ked=badked, code=MtrDex.ECDSA_256r1N) # verfer code not match code - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=pre - prefixer = Prefixer(ked=badked, code=MtrDex.ECDSA_256r1N) # verfer code match code but not pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == False - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked, code=MtrDex.ECDSA_256r1N) # verfer code match code and pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True - - verfer = Verfer(raw=verkey, code=MtrDex.ECDSA_256r1N) - badked = dict(ked) - badked["k"]=[verfer.qb64] - badked["i"]=preN - prefixer = Prefixer(ked=badked) # verfer code match pre code - assert prefixer.qb64 == verfer.qb64 - assert prefixer.verify(ked=badked) == True - assert prefixer.verify(ked=badked, prefixed=True) == True """ Done Test """ -def test_siger(): - """ - Test Siger subclass of Indexer - """ - with pytest.raises(EmptyMaterialError): - siger = Siger() - - qsig64 = ('AACdI8OSQkMJ9r-xigjEByEjIua7LHH3AOJ22PQKqljMhuhcgh9nGRcKnsz5KvKd' - '7K_H9-1298F4Id1DxvIoEmCQ') - #'AAmdI8OSQkMJ9r-xigjEByEjIua7LHH3AOJ22PQKqljMhuhcgh9nGRcKnsz5KvKd7K_H9-1298F4Id1DxvIoEmCQ' - qsig64b = qsig64.encode("utf-8") - assert qsig64b == (b'AACdI8OSQkMJ9r-xigjEByEjIua7LHH3AOJ22PQKqljMhuhcgh9nGR' - b'cKnsz5KvKd7K_H9-1298F4Id1DxvIoEmCQ') - - siger = Siger(qb64b=qsig64b) - assert siger.code == IdrDex.Ed25519_Sig - assert siger.index == 0 - assert siger.ondex == 0 - assert siger.qb64 == qsig64 - assert siger.verfer == None - - - siger = Siger(qb64=qsig64) - assert siger.code == IdrDex.Ed25519_Sig - assert siger.index == 0 - assert siger.ondex == 0 - assert siger.qb64 == qsig64 - assert siger.verfer == None - - - siger = Siger(qb64=qsig64b) # also bytes - assert siger.code == IdrDex.Ed25519_Sig - assert siger.index == 0 - assert siger.ondex == 0 - assert siger.qb64 == qsig64 - assert siger.verfer == None - - - verkey, sigkey = pysodium.crypto_sign_keypair() - verfer = Verfer(raw=verkey) - - siger.verfer = verfer - assert siger.verfer == verfer - - siger = Siger(qb64=qsig64, verfer=verfer) - assert siger.verfer == verfer - - siger = Siger( - raw=b'abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz0123456' - b'789abcdef', code=IdrDex.Ed448_Sig, index=4) - assert siger.qb64 == ('0AEEYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTI' - 'zNDU2Nzg5YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIzNDU2Nzg5YWJjZGVm') - - - """ Done Test """ - def test_saider(): """ Test Saider object """ - # Test class attribute Digest matches DigDex (i.e.DigestCodex) - assert set(Saider.Digests.keys()) == set(code for code in DigDex) code = MtrDex.Blake3_256 - kind = Serials.json + kind = Kinds.json label = Saids.dollar # Test with valid said qb64 @@ -5537,7 +5009,7 @@ def test_saider(): assert saider.verify(sad8, prefixed=True) # verify gets kind from version string if provided when loading from dict - vs = versify(version=Version, kind=Serials.mgpk, size=0) # vaccuous size == 0 + vs = versify(version=Version, kind=Kinds.mgpk, size=0) # vaccuous size == 0 assert vs == 'KERI10MGPK000000_' sad9 = dict(sad4) sad9['v'] = vs @@ -5582,190 +5054,6 @@ def test_saider(): """Done Test""" -def test_serials(): - """ - Test Serializations namedtuple instance Serials - """ - assert Version == Versionage(major=1, minor=0) - - assert isinstance(Serials, Serialage) - - assert Serials.json == 'JSON' - assert Serials.mgpk == 'MGPK' - assert Serials.cbor == 'CBOR' - - assert 'JSON' in Serials - assert 'MGPK' in Serials - assert 'CBOR' in Serials - - assert Vstrings.json == 'KERI10JSON000000_' - assert Vstrings.mgpk == 'KERI10MGPK000000_' - assert Vstrings.cbor == 'KERI10CBOR000000_' - - icp = dict(vs=Vstrings.json, - pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', - sn='0001', - ilk='icp', - dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', - sith=1, - keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], - nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', - toad=0, - wits=[], - cnfg=[], - ) - - rot = dict(vs=Vstrings.json, - pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', - sn='0001', - ilk='rot', - dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', - sith=1, - keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], - nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', - toad=0, - cuts=[], - adds=[], - data=[], - ) - - icps = json.dumps(icp, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - assert len(icps) == 303 - assert icps == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'","sn":"0001","ilk":"icp","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' - b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"wits":[],"cnfg":[]}') - - match = Rever.search(icps) - assert match.group() == Vstrings.json.encode("utf-8") - - rots = json.dumps(rot, separators=(",", ":"), ensure_ascii=False).encode("utf-8") - assert len(rots) == 313 - assert rots == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'","sn":"0001","ilk":"rot","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' - b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"cuts":[],"adds":[],"' - b'data":[]}') - - match = Rever.search(rots) - assert match.group() == Vstrings.json.encode("utf-8") - - icp["vs"] = Vstrings.mgpk - icps = msgpack.dumps(icp) - assert len(icps) == 264 - assert icps == (b'\x8b\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3icp\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' - b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' - b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' - b'CM\xa4toad\x00\xa4wits\x90\xa4cnfg\x90') - - match = Rever.search(icps) - assert match.group() == Vstrings.mgpk.encode("utf-8") - - rot["vs"] = Vstrings.mgpk - rots = msgpack.dumps(rot) - assert len(rots) == 270 - assert rots == (b'\x8c\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' - b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3rot\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' - b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' - b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' - b'CM\xa4toad\x00\xa4cuts\x90\xa4adds\x90\xa4data\x90') - - match = Rever.search(rots) - assert match.group() == Vstrings.mgpk.encode("utf-8") - - icp["vs"] = Vstrings.cbor - icps = cbor.dumps(icp) - assert len(icps) == 264 - assert icps == (b'\xabbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'bsnd0001cilkcicpcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' - b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' - b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dwits\x80dcnfg\x80') - - match = Rever.search(icps) - assert match.group() == Vstrings.cbor.encode("utf-8") - - rot["vs"] = Vstrings.cbor - rots = cbor.dumps(rot) - assert len(rots) == 270 - assert rots == (b'\xacbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' - b'bsnd0001cilkcrotcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' - b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' - b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dcuts\x80dadds\x80ddata\x80') - - match = Rever.search(rots) - assert match.group() == Vstrings.cbor.encode("utf-8") - """Done Test""" - - -def test_versify(): - """ - Test Versify support - """ - vs = versify(kind=Serials.json, size=0) - assert vs == "KERI10JSON000000_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.json - assert version == Version - assert size == 0 - - vs = versify(kind=Serials.json, size=65) - assert vs == "KERI10JSON000041_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.json - assert version == Version - assert size == 65 - - vs = versify(proto=Protos.acdc, kind=Serials.json, size=86) - assert vs == "ACDC10JSON000056_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.acdc - assert kind == Serials.json - assert version == Version - assert size == 86 - - vs = versify(kind=Serials.mgpk, size=0) - assert vs == "KERI10MGPK000000_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.mgpk - assert version == Version - assert size == 0 - - vs = versify(kind=Serials.mgpk, size=65) - assert vs == "KERI10MGPK000041_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.mgpk - assert version == Version - assert size == 65 - - vs = versify(kind=Serials.cbor, size=0) - assert vs == "KERI10CBOR000000_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.cbor - assert version == Version - assert size == 0 - - vs = versify(kind=Serials.cbor, size=65) - assert vs == "KERI10CBOR000041_" - assert len(vs) == VERFULLSIZE - proto, version, kind, size = deversify(vs) - assert proto == Protos.keri - assert kind == Serials.cbor - assert version == Version - assert size == 65 - """End Test""" - def test_tholder(): @@ -6135,23 +5423,170 @@ def test_tholder(): assert not tholder.satisfy(indices=[2, 3, 4]) assert not tholder.satisfy(indices=[]) + # test new nested weighted with Mapping dict with one clause + # 1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1 + tholder = Tholder(sith='[{"1/3":["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]') + assert tholder.weighted + assert tholder.size == 7 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]] + assert tholder.limen ==b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1' + assert tholder.sith ==[{'1/3': ['1/2', '1/2', '1/2']}, '1/3', '1/2', {'1/2': ['1', '1']}] + assert tholder.json == '[{"1/3": ["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]' + assert tholder.num == None + assert tholder.satisfy(indices=[0, 2, 3, 6]) + assert tholder.satisfy(indices=[3, 4, 5]) + assert tholder.satisfy(indices=[1, 2, 3, 4]) + assert tholder.satisfy(indices=[4, 6]) + assert tholder.satisfy(indices=[4, 2, 0, 3]) + assert tholder.satisfy(indices=[0, 0, 1, 2, 1, 5, 6, 3]) + assert not tholder.satisfy(indices=[0, 2, 5]) + assert not tholder.satisfy(indices=[2, 3, 4]) + + tholder = Tholder(limen=b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1') + assert tholder.weighted + assert tholder.size == 7 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]] + assert tholder.limen ==b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1' + assert tholder.sith ==[{'1/3': ['1/2', '1/2', '1/2']}, '1/3', '1/2', {'1/2': ['1', '1']}] + assert tholder.json == '[{"1/3": ["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]' + assert tholder.num == None + + tholder = Tholder(thold=[[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]]) + assert tholder.weighted + assert tholder.size == 7 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), + Fraction(1, 2), + Fraction(1, 2)]), + Fraction(1, 3), + Fraction(1, 2), + (Fraction(1, 2), + [1, 1])]] + assert tholder.limen ==b'4AAIA1s3k1s2v1s2v1s2c1s3c1s2c1s2k1v1' + assert tholder.sith ==[{'1/3': ['1/2', '1/2', '1/2']}, '1/3', '1/2', {'1/2': ['1', '1']}] + assert tholder.json == '[{"1/3": ["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]' + assert tholder.num == None + + with pytest.raises(ValueError): + tholder = Tholder(sith=[{"1/3":["1/3", "1/3", "1/4"]}, "1/3", "1/2", {"1/2": ["1", "1"]}]) + + with pytest.raises(ValueError): + tholder = Tholder(sith=[{"1/3":["1/2", "1/2", "1/2"]}, "1/3", "1/2", {"1/2": ["2/3", "1/4"]}]) + + with pytest.raises(ValueError): + tholder = Tholder(sith=[{"1/5":["1/2", "1/2", "1/2"]}, "1/4", "1/5", {"1/5": ["1", "1"]}]) + + # test new nested weighted with Mapping dict with two clauses + + tholder = Tholder(sith='[[{"1/3":["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ["1/2", {"1/2": ["1", "1"]}]]') + assert tholder.weighted + assert tholder.size == 9 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]] + assert tholder.limen == b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + assert tholder.sith == [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + + assert tholder.json == ('[[{"1/3": ["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ' + '["1/2", ''{"1/2": ["1", "1"]}]]') + assert tholder.num == None + assert tholder.satisfy(indices=[0, 2, 3, 5, 6, 7]) + assert tholder.satisfy(indices=[3, 4, 5, 6, 8]) + assert tholder.satisfy(indices=[1, 2, 3, 4, 6, 7]) + assert tholder.satisfy(indices=[4, 2, 0, 3, 8, 6]) + assert tholder.satisfy(indices=[0, 0, 1, 2, 1, 8, 3, 5, 6, 3]) + assert not tholder.satisfy(indices=[0, 2, 5]) + assert not tholder.satisfy(indices=[6, 7, 8]) + + tholder = Tholder(limen=b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1') + assert tholder.weighted + assert tholder.size == 9 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]] + assert tholder.limen == b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + assert tholder.sith == [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + + assert tholder.json == ('[[{"1/3": ["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ' + '["1/2", ''{"1/2": ["1", "1"]}]]') + assert tholder.num == None + + tholder = Tholder(thold=[[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]]) + assert tholder.weighted + assert tholder.size == 9 + assert tholder.thold == [[(Fraction(1, 3), + [Fraction(1, 2), Fraction(1, 2), Fraction(1, 2)]), + Fraction(1, 2), + (Fraction(1, 2), [1, 1])], + [Fraction(1, 2), (Fraction(1, 2), [1, 1])]] + assert tholder.limen == b'4AAKA1s3k1s2v1s2v1s2c1s2c1s2k1v1a1s2c1s2k1v1' + assert tholder.sith == [[{'1/3': ['1/2', '1/2', '1/2']}, '1/2', {'1/2': ['1', '1']}], + ['1/2', {'1/2': ['1', '1']}]] + + assert tholder.json == ('[[{"1/3": ["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ' + '["1/2", ''{"1/2": ["1", "1"]}]]') + assert tholder.num == None + + with pytest.raises(ValueError): + tholder = Tholder(sith=[[{"1/3":["1/2", "1/2", "1/2"]}, "1/2", {"1/2": ["1", "1"]}], ["1/2", {"1/3": ["1", "1"]}]]) + + with pytest.raises(ValueError): + tholder = Tholder(sith=[[{"1/3":["1/3", "1/4", "1/3"]}, "1/2", {"1/2": ["1", "1"]}], ["1/2", {"1/2": ["1/2", "1/2"]}]]) + """ Done Test """ if __name__ == "__main__": - #test_matter() - #test_counter() + test_mapdom() + test_mapcodex() + test_matter_class() + test_matter() + test_matter_special() + test_tagger() + test_ilker() + test_traitor() + test_verser() + test_diger() + test_texter() + test_bexter() + test_labeler() #test_prodex() - #test_indexer() - #test_number() + test_number() + #test_seqner() #test_siger() - #test_signer() #test_nexter() #test_tholder() - #test_ilks() - #test_labels() - #test_prefixer() - #test_genera() - test_prodex() + test_prefixer() diff --git a/tests/core/test_counting.py b/tests/core/test_counting.py new file mode 100644 index 000000000..7ec6cfbea --- /dev/null +++ b/tests/core/test_counting.py @@ -0,0 +1,1473 @@ +# -*- coding: utf-8 -*- +""" +tests.core.test_counting module + +""" +from dataclasses import dataclass, astuple, asdict +from ordered_set import OrderedSet as oset +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 + +import pytest + + +from keri import kering + +from keri.help import helping +from keri.help.helping import sceil +from keri.help.helping import (intToB64, b64ToInt, codeB64ToB2, codeB2ToB64, + nabSextets) + + + +from keri.core import counting +from keri.core.counting import GenDex, Cizage, Counter, Codens +from keri.core.counting import Versionage, Vrsn_1_0, Vrsn_2_0 + + + +def test_genus_codex(): + """ + Test protocol genera in GenDex as instance of GenusCodex + + """ + + assert asdict(GenDex) == \ + { + 'KERI_ACDC_SPAC': '--AAA', + 'KERI': '--AAA', + 'ACDC': '--AAA', + 'SPAC': '--AAA' + } + + assert '--AAA' in GenDex + assert GenDex.KERI == "--AAA" + assert GenDex.ACDC == "--AAA" + assert GenDex.SPAC == "--AAA" + assert GenDex.KERI_ACDC_SPAC == "--AAA" + assert GenDex.KERI == GenDex.ACDC + + assert hasattr(GenDex, "KERI") + assert hasattr(GenDex, "ACDC") + assert hasattr(GenDex, "SPAC") + assert hasattr(GenDex, "KERI_ACDC_SPAC") + + """End Test""" + + + +def test_codexes_tags(): + """ + Test supporting module attributes + """ + + + assert asdict(counting.CtrDex_1_0) == \ + { + 'ControllerIdxSigs': '-A', + 'WitnessIdxSigs': '-B', + 'NonTransReceiptCouples': '-C', + 'TransReceiptQuadruples': '-D', + 'FirstSeenReplayCouples': '-E', + 'TransIdxSigGroups': '-F', + 'SealSourceCouples': '-G', + 'TransLastIdxSigGroups': '-H', + 'SealSourceTriples': '-I', + 'SadPathSigGroups': '-J', + 'RootSadPathSigGroups': '-K', + 'PathedMaterialGroup': '-L', + 'BigPathedMaterialGroup': '-0L', + 'AttachmentGroup': '-V', + 'ESSRPayloadGroup': '-Z', + 'BigAttachmentGroup': '-0V', + 'KERIACDCGenusVersion': '--AAA' + } + + + assert asdict(counting.CtrDex_2_0) == \ + { + 'GenericGroup': '-A', + 'BigGenericGroup': '-0A', + 'MessageGroup': '-B', + 'BigMessageGroup': '-0B', + 'AttachmentGroup': '-C', + 'BigAttachmentGroup': '-0C', + 'DatagramSegmentGroup': '-D', + 'BigDatagramSegmentGroup': '-0D', + 'ESSRWrapperGroup': '-E', + 'BigESSRWrapperGroup': '-0E', + 'FixedMessageBodyGroup': '-F', + 'BigFixedMessageBodyGroup': '-0F', + 'MapMessageBodyGroup': '-G', + 'BigMapMessageBodyGroup': '-0G', + 'GenericMapGroup': '-H', + 'BigGenericMapGroup': '-0H', + 'GenericListGroup': '-I', + 'BigGenericListGroup': '-0I', + 'ControllerIdxSigs': '-J', + 'BigControllerIdxSigs': '-0J', + 'WitnessIdxSigs': '-K', + 'BigWitnessIdxSigs': '-0K', + 'NonTransReceiptCouples': '-L', + 'BigNonTransReceiptCouples': '-0L', + 'TransReceiptQuadruples': '-M', + 'BigTransReceiptQuadruples': '-0M', + 'FirstSeenReplayCouples': '-N', + 'BigFirstSeenReplayCouples': '-0N', + 'TransIdxSigGroups': '-O', + 'BigTransIdxSigGroups': '-0O', + 'TransLastIdxSigGroups': '-P', + 'BigTransLastIdxSigGroups': '-0P', + 'SealSourceCouples': '-Q', + 'BigSealSourceCouples': '-0Q', + 'SealSourceTriples': '-R', + 'BigSealSourceTriples': '-0R', + 'PathedMaterialGroup': '-S', + 'BigPathedMaterialGroup': '-0S', + 'SadPathSigGroups': '-T', + 'BigSadPathSigGroups': '-0T', + 'RootSadPathSigGroups': '-U', + 'BigRootSadPathSigGroups': '-0U', + 'DigestSealSingles': '-V', + 'BigDigestSealSingles': '-0V', + 'MerkleRootSealSingles': '-W', + 'BigMerkleRootSealSingles': '-0W', + 'BackerRegistrarSealCouples': '-X', + 'BigBackerRegistrarSealCouples': '-0X', + 'SealSourceLastSingles': '-Y', + 'BigSealSourceLastSingles': '-0Y', + 'ESSRPayloadGroup': '-Z', + 'BigESSRPayloadGroup': '-0Z', + 'KERIACDCGenusVersion': '--AAA' + } + + assert counting.CodeNames == ( + 'GenericGroup', + 'BigGenericGroup', + 'MessageGroup', + 'BigMessageGroup', + 'AttachmentGroup', + 'BigAttachmentGroup', + 'DatagramSegmentGroup', + 'BigDatagramSegmentGroup', + 'ESSRWrapperGroup', + 'BigESSRWrapperGroup', + 'FixedMessageBodyGroup', + 'BigFixedMessageBodyGroup', + 'MapMessageBodyGroup', + 'BigMapMessageBodyGroup', + 'GenericMapGroup', + 'BigGenericMapGroup', + 'GenericListGroup', + 'BigGenericListGroup', + 'ControllerIdxSigs', + 'BigControllerIdxSigs', + 'WitnessIdxSigs', + 'BigWitnessIdxSigs', + 'NonTransReceiptCouples', + 'BigNonTransReceiptCouples', + 'TransReceiptQuadruples', + 'BigTransReceiptQuadruples', + 'FirstSeenReplayCouples', + 'BigFirstSeenReplayCouples', + 'TransIdxSigGroups', + 'BigTransIdxSigGroups', + 'TransLastIdxSigGroups', + 'BigTransLastIdxSigGroups', + 'SealSourceCouples', + 'BigSealSourceCouples', + 'SealSourceTriples', + 'BigSealSourceTriples', + 'PathedMaterialGroup', + 'BigPathedMaterialGroup', + 'SadPathSigGroups', + 'BigSadPathSigGroups', + 'RootSadPathSigGroups', + 'BigRootSadPathSigGroups', + 'DigestSealSingles', + 'BigDigestSealSingles', + 'MerkleRootSealSingles', + 'BigMerkleRootSealSingles', + 'BackerRegistrarSealCouples', + 'BigBackerRegistrarSealCouples', + 'SealSourceLastSingles', + 'BigSealSourceLastSingles', + 'ESSRPayloadGroup', + 'BigESSRPayloadGroup', + 'KERIACDCGenusVersion' + ) + assert 'ControllerIdxSigs' in counting.CodeNames + + assert counting.Codens == counting.Codenage( + GenericGroup='GenericGroup', + BigGenericGroup='BigGenericGroup', + MessageGroup='MessageGroup', + BigMessageGroup='BigMessageGroup', + AttachmentGroup='AttachmentGroup', + BigAttachmentGroup='BigAttachmentGroup', + DatagramSegmentGroup='DatagramSegmentGroup', + BigDatagramSegmentGroup='BigDatagramSegmentGroup', + ESSRWrapperGroup='ESSRWrapperGroup', + BigESSRWrapperGroup='BigESSRWrapperGroup', + FixedMessageBodyGroup='FixedMessageBodyGroup', + BigFixedMessageBodyGroup='BigFixedMessageBodyGroup', + MapMessageBodyGroup='MapMessageBodyGroup', + BigMapMessageBodyGroup='BigMapMessageBodyGroup', + GenericMapGroup='GenericMapGroup', + BigGenericMapGroup='BigGenericMapGroup', + GenericListGroup='GenericListGroup', + BigGenericListGroup='BigGenericListGroup', + ControllerIdxSigs='ControllerIdxSigs', + BigControllerIdxSigs='BigControllerIdxSigs', + WitnessIdxSigs='WitnessIdxSigs', + BigWitnessIdxSigs='BigWitnessIdxSigs', + NonTransReceiptCouples='NonTransReceiptCouples', + BigNonTransReceiptCouples='BigNonTransReceiptCouples', + TransReceiptQuadruples='TransReceiptQuadruples', + BigTransReceiptQuadruples='BigTransReceiptQuadruples', + FirstSeenReplayCouples='FirstSeenReplayCouples', + BigFirstSeenReplayCouples='BigFirstSeenReplayCouples', + TransIdxSigGroups='TransIdxSigGroups', + BigTransIdxSigGroups='BigTransIdxSigGroups', + TransLastIdxSigGroups='TransLastIdxSigGroups', + BigTransLastIdxSigGroups='BigTransLastIdxSigGroups', + SealSourceCouples='SealSourceCouples', + BigSealSourceCouples='BigSealSourceCouples', + SealSourceTriples='SealSourceTriples', + BigSealSourceTriples='BigSealSourceTriples', + PathedMaterialGroup='PathedMaterialGroup', + BigPathedMaterialGroup='BigPathedMaterialGroup', + SadPathSigGroups='SadPathSigGroups', + BigSadPathSigGroups='BigSadPathSigGroups', + RootSadPathSigGroups='RootSadPathSigGroups', + BigRootSadPathSigGroups='BigRootSadPathSigGroups', + DigestSealSingles='DigestSealSingles', + BigDigestSealSingles='BigDigestSealSingles', + MerkleRootSealSingles='MerkleRootSealSingles', + BigMerkleRootSealSingles='BigMerkleRootSealSingles', + BackerRegistrarSealCouples='BackerRegistrarSealCouples', + BigBackerRegistrarSealCouples='BigBackerRegistrarSealCouples', + SealSourceLastSingles='SealSourceLastSingles', + BigSealSourceLastSingles='BigSealSourceLastSingles', + ESSRPayloadGroup='ESSRPayloadGroup', + BigESSRPayloadGroup='BigESSRPayloadGroup', + KERIACDCGenusVersion='KERIACDCGenusVersion' + ) + + assert counting.Codens.ControllerIdxSigs == 'ControllerIdxSigs' + + + + assert asdict(counting.SealDex_2_0) == \ + { + 'SealSourceCouples': '-Q', + 'BigSealSourceCouples': '-0Q', + 'SealSourceTriples': '-R', + 'BigSealSourceTriples': '-0R', + 'DigestSealSingles': '-V', + 'BigDigestSealSingles': '-0V', + 'MerkleRootSealSingles': '-W', + 'BigMerkleRootSealSingles': '-0W', + 'BackerRegistrarSealCouples': '-X', + 'BigBackerRegistrarSealCouples': '-0X', + 'SealSourceLastSingles': '-Y', + 'BigSealSourceLastSingles': '-0Y', + } + + + + """End Test""" + + +def test_counter_class(): + """ + Test Counter class variables + """ + # test class attributes + + assert Counter.Codes == \ + { + Vrsn_1_0.major: \ + { + Vrsn_1_0.minor: counting.CtrDex_1_0, + }, + Vrsn_2_0.major: \ + { + Vrsn_2_0.minor: counting.CtrDex_2_0, + }, + } + + assert Counter.Names == \ + {1: + {0: + { + '-A': 'ControllerIdxSigs', + '-B': 'WitnessIdxSigs', + '-C': 'NonTransReceiptCouples', + '-D': 'TransReceiptQuadruples', + '-E': 'FirstSeenReplayCouples', + '-F': 'TransIdxSigGroups', + '-G': 'SealSourceCouples', + '-H': 'TransLastIdxSigGroups', + '-I': 'SealSourceTriples', + '-J': 'SadPathSigGroups', + '-K': 'RootSadPathSigGroups', + '-L': 'PathedMaterialGroup', + '-0L': 'BigPathedMaterialGroup', + '-V': 'AttachmentGroup', + '-0V': 'BigAttachmentGroup', + '-Z': 'ESSRPayloadGroup', + '--AAA': 'KERIACDCGenusVersion' + } + }, + 2: + {0: + { + '-A': 'GenericGroup', + '-0A': 'BigGenericGroup', + '-B': 'MessageGroup', + '-0B': 'BigMessageGroup', + '-C': 'AttachmentGroup', + '-0C': 'BigAttachmentGroup', + '-D': 'DatagramSegmentGroup', + '-0D': 'BigDatagramSegmentGroup', + '-E': 'ESSRWrapperGroup', + '-0E': 'BigESSRWrapperGroup', + '-F': 'FixedMessageBodyGroup', + '-0F': 'BigFixedMessageBodyGroup', + '-G': 'MapMessageBodyGroup', + '-0G': 'BigMapMessageBodyGroup', + '-H': 'GenericMapGroup', + '-0H': 'BigGenericMapGroup', + '-I': 'GenericListGroup', + '-0I': 'BigGenericListGroup', + '-J': 'ControllerIdxSigs', + '-0J': 'BigControllerIdxSigs', + '-K': 'WitnessIdxSigs', + '-0K': 'BigWitnessIdxSigs', + '-L': 'NonTransReceiptCouples', + '-0L': 'BigNonTransReceiptCouples', + '-M': 'TransReceiptQuadruples', + '-0M': 'BigTransReceiptQuadruples', + '-N': 'FirstSeenReplayCouples', + '-0N': 'BigFirstSeenReplayCouples', + '-O': 'TransIdxSigGroups', + '-0O': 'BigTransIdxSigGroups', + '-P': 'TransLastIdxSigGroups', + '-0P': 'BigTransLastIdxSigGroups', + '-Q': 'SealSourceCouples', + '-0Q': 'BigSealSourceCouples', + '-R': 'SealSourceTriples', + '-0R': 'BigSealSourceTriples', + '-S': 'PathedMaterialGroup', + '-0S': 'BigPathedMaterialGroup', + '-T': 'SadPathSigGroups', + '-0T': 'BigSadPathSigGroups', + '-U': 'RootSadPathSigGroups', + '-0U': 'BigRootSadPathSigGroups', + '-V': 'DigestSealSingles', + '-0V': 'BigDigestSealSingles', + '-W': 'MerkleRootSealSingles', + '-0W': 'BigMerkleRootSealSingles', + '-X': 'BackerRegistrarSealCouples', + '-0X': 'BigBackerRegistrarSealCouples', + '-Y': 'SealSourceLastSingles', + '-0Y': 'BigSealSourceLastSingles', + '-Z': 'ESSRPayloadGroup', + '-0Z': 'BigESSRPayloadGroup', + '--AAA': 'KERIACDCGenusVersion' + } + } + } + + + # Codes table with sizes of code (hard) and full primitive material + assert Counter.Sizes == \ + { + 1: + { + 0: + { + '-A': Cizage(hs=2, ss=2, fs=4), + '-B': Cizage(hs=2, ss=2, fs=4), + '-C': Cizage(hs=2, ss=2, fs=4), + '-D': Cizage(hs=2, ss=2, fs=4), + '-E': Cizage(hs=2, ss=2, fs=4), + '-F': Cizage(hs=2, ss=2, fs=4), + '-G': Cizage(hs=2, ss=2, fs=4), + '-H': Cizage(hs=2, ss=2, fs=4), + '-I': Cizage(hs=2, ss=2, fs=4), + '-J': Cizage(hs=2, ss=2, fs=4), + '-K': Cizage(hs=2, ss=2, fs=4), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '--AAA': Cizage(hs=5, ss=3, fs=8) + } + }, + 2: + { + 0: + { + '-A': Cizage(hs=2, ss=2, fs=4), + '-0A': Cizage(hs=3, ss=5, fs=8), + '-B': Cizage(hs=2, ss=2, fs=4), + '-0B': Cizage(hs=3, ss=5, fs=8), + '-C': Cizage(hs=2, ss=2, fs=4), + '-0C': Cizage(hs=3, ss=5, fs=8), + '-D': Cizage(hs=2, ss=2, fs=4), + '-0D': Cizage(hs=3, ss=5, fs=8), + '-E': Cizage(hs=2, ss=2, fs=4), + '-0E': Cizage(hs=3, ss=5, fs=8), + '-F': Cizage(hs=2, ss=2, fs=4), + '-0F': Cizage(hs=3, ss=5, fs=8), + '-G': Cizage(hs=2, ss=2, fs=4), + '-0G': Cizage(hs=3, ss=5, fs=8), + '-H': Cizage(hs=2, ss=2, fs=4), + '-0H': Cizage(hs=3, ss=5, fs=8), + '-I': Cizage(hs=2, ss=2, fs=4), + '-0I': Cizage(hs=3, ss=5, fs=8), + '-J': Cizage(hs=2, ss=2, fs=4), + '-0J': Cizage(hs=3, ss=5, fs=8), + '-K': Cizage(hs=2, ss=2, fs=4), + '-0K': Cizage(hs=3, ss=5, fs=8), + '-L': Cizage(hs=2, ss=2, fs=4), + '-0L': Cizage(hs=3, ss=5, fs=8), + '-M': Cizage(hs=2, ss=2, fs=4), + '-0M': Cizage(hs=3, ss=5, fs=8), + '-N': Cizage(hs=2, ss=2, fs=4), + '-0N': Cizage(hs=3, ss=5, fs=8), + '-O': Cizage(hs=2, ss=2, fs=4), + '-0O': Cizage(hs=3, ss=5, fs=8), + '-P': Cizage(hs=2, ss=2, fs=4), + '-0P': Cizage(hs=3, ss=5, fs=8), + '-Q': Cizage(hs=2, ss=2, fs=4), + '-0Q': Cizage(hs=3, ss=5, fs=8), + '-R': Cizage(hs=2, ss=2, fs=4), + '-0R': Cizage(hs=3, ss=5, fs=8), + '-S': Cizage(hs=2, ss=2, fs=4), + '-0S': Cizage(hs=3, ss=5, fs=8), + '-T': Cizage(hs=2, ss=2, fs=4), + '-0T': Cizage(hs=3, ss=5, fs=8), + '-U': Cizage(hs=2, ss=2, fs=4), + '-0U': Cizage(hs=3, ss=5, fs=8), + '-V': Cizage(hs=2, ss=2, fs=4), + '-0V': Cizage(hs=3, ss=5, fs=8), + '-W': Cizage(hs=2, ss=2, fs=4), + '-0W': Cizage(hs=3, ss=5, fs=8), + '-X': Cizage(hs=2, ss=2, fs=4), + '-0X': Cizage(hs=3, ss=5, fs=8), + '-Y': Cizage(hs=2, ss=2, fs=4), + '-0Y': Cizage(hs=3, ss=5, fs=8), + '-Z': Cizage(hs=2, ss=2, fs=4), + '-0Z': Cizage(hs=3, ss=5, fs=8), + '--AAA': Cizage(hs=5, ss=3, fs=8) + } + } + } + + # Ensure there is an entry in Sizes for each entry in Codes + assert Counter.Codes.keys() == Counter.Sizes.keys() + for majorc, majors in zip(Counter.Codes.items(), Counter.Sizes.items(), strict=True): + assert majorc[0] == majors[0] # major version, keys match + for minorc, minors in zip(majorc[1].items(), majors[1].items(), strict=True): + assert minorc[0] == minors[0] # minor version keys match + for code, size in zip(asdict(minorc[1]).items(), minors[1].items(), strict=True): + code[0] == size[0] # code and size keys match + + + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].hs == 2 # hard size + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].ss == 2 # soft size + assert Counter.Sizes[Vrsn_1_0.major][Vrsn_1_0.minor]['-A'].fs == 4 # full size + + assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].hs == 3 # hard size + assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].ss == 5 # soft size + assert Counter.Sizes[Vrsn_2_0.major][Vrsn_2_0.minor]['-0A'].fs == 8 # full size + + # first character of code with hard size of code + assert Counter.Hards == \ + { + '-A': 2, '-B': 2, '-C': 2, '-D': 2, '-E': 2, '-F': 2, '-G': 2, '-H': 2, '-I': 2, + '-J': 2, '-K': 2, '-L': 2, '-M': 2, '-N': 2, '-O': 2, '-P': 2, '-Q': 2, '-R': 2, + '-S': 2, '-T': 2, '-U': 2, '-V': 2, '-W': 2, '-X': 2, '-Y': 2, '-Z': 2, + '-a': 2, '-b': 2, '-c': 2, '-d': 2, '-e': 2, '-f': 2, '-g': 2, '-h': 2, '-i': 2, + '-j': 2, '-k': 2, '-l': 2, '-m': 2, '-n': 2, '-o': 2, '-p': 2, '-q': 2, '-r': 2, + '-s': 2, '-t': 2, '-u': 2, '-v': 2, '-w': 2, '-x': 2, '-y': 2, '-z': 2, + '-0': 3, '--': 5, + } + + + + # verify first hs Sizes matches hs in Codes for same first char + for vmajor in Counter.Sizes.values(): + for vminor in vmajor.values(): + for key, val in vminor.items(): # size table items + assert Counter.Hards[key[:2]] == val.hs + + # verify all Codes have hs >= 2 and ss > 0 and fs = hs + ss and not fs % 4 + for vmajor in Counter.Sizes.values(): + for vminor in vmajor.values(): + for val in vminor.values(): # size table values + assert val.hs >= 2 and val.ss > 0 and val.hs + val.ss == val.fs and not val.fs % 4 + + # Bizes maps bytes of sextet of decoded first character of code with hard size of code + # verify equivalents of items for Sizes and Bizes + for skey, sval in Counter.Hards.items(): + ckey = codeB64ToB2(skey) + assert Counter.Bards[ckey] == sval + + # test Counter static methods + assert Counter.verToB64() == "AAA" + + assert Counter.b64ToVer('AAA') == Versionage(major=0, minor=0) + assert Counter.b64ToVer('AAA', texted=True) == "0.0" + assert Counter.b64ToVer('BBC', texted=True) == "1.66" + assert Counter.b64ToVer('bbc', texted=True) == '27.1756' + + with pytest.raises(ValueError): + Counter.b64ToVer("!AA") + + with pytest.raises(ValueError): + Counter.b64ToVer("AA#") + + assert Counter.b64ToVer(Counter.verToB64(text='1.1'), texted=True) == "1.1" + assert Counter.verToB64(text=Counter.b64ToVer('BAB', texted=True)) == "BAB" + assert Counter.b64ToVer(Counter.verToB64(text='12.2345'), texted=True) == '12.2345' + assert Counter.verToB64(text='12.2345') == 'Mkp' + assert Counter.verToB64(text=Counter.b64ToVer('Mkp', texted=True)) == 'Mkp' + + assert Counter.verToB64(counting.Vrsn_1_0) == "BAA" + assert Counter.verToB64(counting.Vrsn_2_0) == "CAA" + + assert Counter.verToB64(text="1.2") == "BAC" + + assert Counter.verToB64(major=1) == "BAA" + assert Counter.verToB64(minor=1) == "AAB" + assert Counter.verToB64(major=3, minor=4) == "DAE" + + # test defaults for missing parts in string version + assert Counter.verToB64(text="1.1") == "BAB" + assert Counter.verToB64(text="1.1.") == "BAB" # ignores extra parts + assert Counter.verToB64(text="1.1.0") == "BAB" # ignores extra parts + assert Counter.verToB64(text="1.") == "BAA" + assert Counter.verToB64(text="1") == "BAA" + assert Counter.verToB64(text="1.2") == "BAC" + assert Counter.verToB64(text=".") == "AAA" + assert Counter.verToB64(text="1.3") == "BAD" + assert Counter.verToB64(text="4", major=1, minor=2) == "EAC" + + with pytest.raises(ValueError): + Counter.verToB64(text="64.0") + with pytest.raises(ValueError): + Counter.verToB64(text="63.4096") + with pytest.raises(ValueError): + Counter.verToB64(text="-1.0") + with pytest.raises(ValueError): + Counter.verToB64(text="0.-1") + with pytest.raises(ValueError): + Counter.verToB64(major=64) + with pytest.raises(ValueError): + Counter.verToB64(minor=4096) + with pytest.raises(ValueError): + Counter.verToB64(major=-1) + with pytest.raises(ValueError): + Counter.verToB64(minor=-1) + + + """ Done Test """ + +def test_counter_v1(): + """ + test Counter instances for verision 1.0 code tables + """ + # version 1_0 tests + CtrDex = Counter.Codes[Vrsn_1_0.major][Vrsn_1_0.minor] # set CtrDex to Vrsn_1_0 + + # test Counter instances + with pytest.raises(kering.EmptyMaterialError): + counter = Counter(gvrsn=Vrsn_1_0) + + # create code manually + count = 1 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.ControllerIdxSigs == counter.hard + assert counter.name == "ControllerIdxSigs" + + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + assert counter.fullSize == 4 + assert counter.soft =='AB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 + assert counter.codes == counting.CtrDex_1_0 + #assert counter.tags == counting.Tags_1_0 + assert counter.sizes == Counter.Sizes[1][0] + + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + # test keyword buth with code name + counter = Counter(code=Codens.ControllerIdxSigs, + count=count, + gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(code=CtrDex.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default count = 1 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + # test truncates extra bytes from qb64 parameter + longqsc64 = qsc + "ABCD" + counter = Counter(qb64=longqsc64, gvrsn=Vrsn_1_0) + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb64 parameter + shortqsc64 = qsc[:-1] # too short + with pytest.raises(kering.ShortageError): + counter = Counter(qb64=shortqsc64, gvrsn=Vrsn_1_0) + + # test truncates extra bytes from qb2 parameter + longqscb2 = qscb2 + bytearray([1, 2, 3, 4, 5]) # extra bytes in size + counter = Counter(qb2=longqscb2, gvrsn=Vrsn_1_0) + assert counter.qb2 == qscb2 + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb2 parameter + shortqscb2 = qscb2[:-4] # too few bytes in size + with pytest.raises(kering.ShortageError): + counter = Counter(qb2=shortqscb2, gvrsn=Vrsn_1_0) + + # test with non-zero count=5 + count = 5 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAF' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + # test with big codes index=100024000 + count = 100024000 + qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) + assert qsc == '-0VF9j7A' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigAttachmentGroup, count=count, gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str + assert counter.code == CtrDex.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes + assert counter.code == CtrDex.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 + assert counter.code == CtrDex.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + # Test ._bexfil + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # + code = counter.code + count = counter.count + qb2 = counter.qb2 + counter._bexfil(qb2) + assert counter.code == code + assert counter.name == "BigAttachmentGroup" + assert counter.count == count + assert counter.qb64 == qsc + assert counter.qb2 == qb2 + assert counter.version == Vrsn_1_0 + + # Test ._binfil + test = counter._binfil() + assert test == qb2 + + # test BigPathedMaterialGroup with big codes index=100024000 + count = 100024000 + qsc = CtrDex.BigPathedMaterialGroup + intToB64(count, l=5) + assert qsc == '-0LF9j7A' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigPathedMaterialGroup, count=count, gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_1_0) # test with bytes not str + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # test with str not bytes + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_1_0) # test with qb2 + assert counter.code == CtrDex.BigPathedMaterialGroup + assert counter.name == "BigPathedMaterialGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + # Test ._bexfil + counter = Counter(qb64=qsc, gvrsn=Vrsn_1_0) # + code = counter.code + count = counter.count + qb2 = counter.qb2 + counter._bexfil(qb2) + assert counter.code == code + assert counter.name == "BigPathedMaterialGroup" + assert counter.count == count + assert counter.qb64 == qsc + assert counter.qb2 == qb2 + assert counter.version == Vrsn_1_0 + + # Test ._binfil + test = counter._binfil() + assert test == qb2 + + # Test with strip + # create code manually + count = 1 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-AAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + # strip ignored if qb64 + counter = Counter(qb64=qsc, strip=True, gvrsn=Vrsn_1_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + ims = bytearray(qscb) # test with qb64b + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) # strip + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + ims = bytearray(qscb2) # test with qb2 + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + # test with longer ims for qb64b + extra = b"ABCD" + ims = bytearray(qscb + b"ABCD") + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) + assert counter.qb64b == qscb + assert len(counter.qb64b) == counter.sizes[counter.code].fs + assert ims == extra + + # test with longer ims for qb2 + extra = bytearray([1, 2, 3, 4, 5]) + ims = bytearray(qscb2) + extra + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) + assert counter.qb2 == qscb2 + assert len(counter.qb2) == counter.sizes[counter.code].fs * 3 // 4 + assert ims == extra + + # raises error if not bytearray + + ims = bytes(qscb) # test with qb64b + with pytest.raises(TypeError): + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) # strip + + ims = bytes(qscb2) # test with qb2 + with pytest.raises(TypeError): + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) + + # test with big codes index=1024 + count = 1024 + qsc = CtrDex.BigAttachmentGroup + intToB64(count, l=5) + assert qsc == '-0VAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + ims = bytearray(qscb) + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_1_0) # test with bytes not str + assert counter.code == CtrDex.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + assert counter.fullSize == 8 + assert not ims + + ims = bytearray(qscb2) + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_1_0) # test with qb2 + assert counter.code == CtrDex.BigAttachmentGroup + assert counter.name == "BigAttachmentGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + assert counter.fullSize == 8 + assert not ims + + # test protocol genus with CESR protocol genus version + # test with big codes index=1024 + genverint = 0 + genver = intToB64(genverint, l=3) + assert genver == 'AAA' + assert genverint == b64ToInt(genver) + qsc = CtrDex.KERIACDCGenusVersion + genver + assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + count=genverint, + gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + assert counter.codes == counting.CtrDex_1_0 + #assert counter.tags == counting.Tags_1_0 + assert counter.sizes == Counter.Sizes[1][0] + + + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + countB64=genver, + gvrsn=Vrsn_1_0) + assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_1_0 + + """End Test""" + +def test_counter_v2(): + """ + test Counter instances for verision 2.0 code tables + """ + # version 2_0 tests default version is Version + CtrDex = Counter.Codes[Vrsn_2_0.major][Vrsn_2_0.minor] # set CtrDex to Vrsn_2_0 + + # test Counter instances + with pytest.raises(kering.EmptyMaterialError): + counter = Counter() + + # create code manually + count = 1 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-JAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + # default version and default count = 1 + counter = Counter(code=CtrDex.ControllerIdxSigs) + assert counter.code == CtrDex.ControllerIdxSigs == counter.hard + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert counter.fullSize == 4 + assert counter.soft =='AB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 + assert counter.codes == counting.CtrDex_2_0 + #assert counter.tags == counting.Tags_2_0 + assert counter.sizes == Counter.Sizes[2][0] + + + + # default count = 1 + counter = Counter(code=CtrDex.ControllerIdxSigs, gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + # test truncates extra bytes from qb64 parameter + longqsc64 = qsc + "ABCD" + counter = Counter(qb64=longqsc64, gvrsn=Vrsn_2_0) + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb64 parameter + shortqsc64 = qsc[:-1] # too short + with pytest.raises(kering.ShortageError): + counter = Counter(qb64=shortqsc64, gvrsn=Vrsn_2_0) + + # test truncates extra bytes from qb2 parameter + longqscb2 = qscb2 + bytearray([1, 2, 3, 4, 5]) # extra bytes in size + counter = Counter(qb2=longqscb2, gvrsn=Vrsn_2_0) + assert counter.qb2 == qscb2 + assert len(counter.qb64) == counter.sizes[counter.code].fs + + # test raises ShortageError if not enough bytes in qb2 parameter + shortqscb2 = qscb2[:-4] # too few bytes in size + with pytest.raises(kering.ShortageError): + counter = Counter(qb2=shortqscb2, gvrsn=Vrsn_2_0) + + # test with non-zero count=5 + count = 5 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-JAF' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(Codens.ControllerIdxSigs, + count=count, + gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + # test keyword with code name + counter = Counter(code=Codens.ControllerIdxSigs, + count=count, + gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(code=CtrDex.ControllerIdxSigs, count=count, gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + # Test ._bexfil + counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) + code = counter.code + count = counter.count + qb2 = counter.qb2 + counter._bexfil(qb2) + assert counter.code == code + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64 == qsc + assert counter.qb2 == qb2 + assert counter.version == Vrsn_2_0 + + # Test ._binfil + test = counter._binfil() + assert test == qb2 + + # Test with strip + # create code manually + count = 1 + qsc = CtrDex.ControllerIdxSigs + intToB64(count, l=2) + assert qsc == '-JAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + # strip ignored if qb64 + counter = Counter(qb64=qsc, strip=True, gvrsn=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + ims = bytearray(qscb) # test with qb64b + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # strip + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + ims = bytearray(qscb2) # test with qb2 + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) + assert not ims # deleted + assert counter.code == CtrDex.ControllerIdxSigs + assert counter.name == "ControllerIdxSigs" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + # test with longer ims for qb64b + extra = b"ABCD" + ims = bytearray(qscb + b"ABCD") + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) + assert counter.qb64b == qscb + assert len(counter.qb64b) == counter.sizes[counter.code].fs + assert ims == extra + + # test with longer ims for qb2 + extra = bytearray([1, 2, 3, 4, 5]) + ims = bytearray(qscb2) + extra + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) + assert counter.qb2 == qscb2 + assert len(counter.qb2) == counter.sizes[counter.code].fs * 3 // 4 + assert ims == extra + + # raises error if not bytearray + + ims = bytes(qscb) # test with qb64b + with pytest.raises(TypeError): + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # strip + + ims = bytes(qscb2) # test with qb2 + with pytest.raises(TypeError): + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) + + # test with big codes count=1024 + count = 1024 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigGenericGroup, count=count, gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + # test ims with big codes count=1024 + count = 1024 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAAAQA' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + ims = bytearray(qscb) + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert not ims + + ims = bytearray(qscb2) + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert not ims + + # test with big codes count=8193 + count = 8193 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAACAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.BigGenericGroup, count=count, gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup == counter.hard + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert counter.fullSize == 8 + assert counter.soft == 'AACAB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 + + counter = Counter(qb64b=qscb, gvrsn=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb64=qsc, gvrsn=Vrsn_2_0) # test with str not bytes + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + counter = Counter(qb2=qscb2, gvrsn=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + # test ims with big codes count=8193 + count = 8193 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAACAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + ims = bytearray(qscb) + counter = Counter(qb64b=ims, strip=True, gvrsn=Vrsn_2_0) # test with bytes not str + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert counter.fullSize == 8 + assert not ims + + ims = bytearray(qscb2) + counter = Counter(qb2=ims, strip=True, gvrsn=Vrsn_2_0) # test with qb2 + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert not ims + + # test with promotion from small to big codes with count=8193 + count = 8193 + qsc = CtrDex.BigGenericGroup + intToB64(count, l=5) + assert qsc == '-0AAACAB' + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.GenericGroup, count=count, gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup == counter.hard + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert counter.soft =='AACAB' + assert counter.both == qsc == counter.hard + counter.soft == counter.qb64 + + counter = Counter(Codens.GenericGroup, count=count, gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.BigGenericGroup + assert counter.name == "BigGenericGroup" + assert counter.count == count + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + # test protocol genus with CESR version + genverint = 0 + genver = intToB64(genverint, l=3) + assert genver == 'AAA' + assert genverint == b64ToInt(genver) + qsc = CtrDex.KERIACDCGenusVersion + genver + assert qsc == '--AAAAAA' # keri Cesr version 0.0.0 + qscb = qsc.encode("utf-8") + qscb2 = decodeB64(qscb) + + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + count=genverint, + gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + assert counter.fullSize == 8 + assert counter.codes == counting.CtrDex_2_0 + #assert counter.tags == counting.Tags_2_0 + assert counter.sizes == Counter.Sizes[2][0] + + counter = Counter(code=CtrDex.KERIACDCGenusVersion, + countB64=genver, + gvrsn=Vrsn_2_0) + assert counter.code == CtrDex.KERIACDCGenusVersion + assert counter.name == "KERIACDCGenusVersion" + assert counter.count == genverint + assert counter.countToB64(l=3) == genver + assert counter.countToB64() == genver # default length + assert counter.qb64b == qscb + assert counter.qb64 == qsc + assert counter.qb2 == qscb2 + assert counter.version == Vrsn_2_0 + + """End Test""" + + +if __name__ == "__main__": + test_genus_codex() + test_codexes_tags() + test_counter_class() + test_counter_v1() + test_counter_v2() + + diff --git a/tests/core/test_delegating.py b/tests/core/test_delegating.py index deff595ad..269b54f7a 100644 --- a/tests/core/test_delegating.py +++ b/tests/core/test_delegating.py @@ -3,12 +3,18 @@ tests delegation primaily from keri.core.eventing """ +import logging import os from keri import help -from keri.app import keeping + +from keri import kering, core from keri.core import coring, eventing, parsing + +from keri.app import keeping, habbing + from keri.db import dbing, basing +from keri.db.dbing import snKey logger = help.ogler.getLogger() @@ -20,8 +26,8 @@ def test_delegation(): """ # bob is the delegator del is bob's delegate - bobSalt = coring.Salter(raw=b'0123456789abcdef').qb64 - delSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 + delSalt = core.Salter(raw=b'abcdef0123456789').qb64 with (basing.openDB(name="bob") as bobDB, \ keeping.openKS(name="bob") as bobKS, \ @@ -50,7 +56,8 @@ def test_delegation(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -102,8 +109,8 @@ def test_delegation(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -130,13 +137,13 @@ def test_delegation(): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) @@ -196,8 +203,8 @@ def test_delegation(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -224,13 +231,13 @@ def test_delegation(): sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) @@ -276,6 +283,498 @@ def test_delegation(): """End Test""" +def test_delegation_supersede(): + """ + Test superseding delegation rules + + Three level delegation + top is at top or root level with witness wop. top is not delegated + mid is at mid-level with witness wid. mid is delegated from top + bot is at bottom level with witness wot. bot is delegated from mid + + + def test_load_event(mockHelpingNowUTC): + with habbing.openHby(name="tor", base="test") as torHby, \ + habbing.openHby(name="wil", base="test") as wilHby, \ + habbing.openHby(name="wan", base="test") as wanHby, \ + habbing.openHby(name="tee", base="test") as teeHby: + + + + # Create Wan the witness + wanHab = wanHby.makeHab(name="wan", transferable=False) + assert wanHab.pre == "BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP" + msg = wanHab.makeOwnEvent(sn=0) + parsing.Parser().parse(ims=msg, kvy=torKvy) + + # Create Wil the witness, we'll use him later + wilHab = wilHby.makeHab(name="wil", transferable=False) + + # Create Tor the delegaTOR and pass to witness Wan + torHab = torHby.makeHab(name="tor", icount=1, isith='1', ncount=1, nsith='1', wits=[wanHab.pre], toad=1) + assert torHab.pre == "EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP" + torIcp = torHab.makeOwnEvent(sn=0) + + wanKvy = Kevery(db=wanHby.db, lax=False, local=False) # remote events + torKvy = Kevery(db=torHby.db, lax=False, local=False) # remote events + + + """ + topSalt = core.Salter(raw=b'0123456789abcdef').qb64 + wopSalt = core.Salter(raw=b'0123456789abcdef').qb64 + midSalt = core.Salter(raw=b'abcdef0123456789').qb64 + widSalt = core.Salter(raw=b'abcdef0123456789').qb64 + botSalt = core.Salter(raw=b'zyxwvutsrponmlkj').qb64 + wotSalt = core.Salter(raw=b'zyxwvutsrponmlkj').qb64 + + with (habbing.openHby(name="top", base="test", salt=topSalt) as topHby, + habbing.openHby(name="wop", base="test", salt=wopSalt) as wopHby, + habbing.openHby(name="mid", base="test", salt=midSalt) as midHby, + habbing.openHby(name="wid", base="test", salt=widSalt) as widHby, + habbing.openHby(name="bot", base="test", salt=botSalt) as botHby, + habbing.openHby(name="wot", base="test", salt=wotSalt) as wotHby): + + # Create witness wop and controller top + wopHab = wopHby.makeHab(name="wop", transferable=False) # witness nontrans + # makehab also enters inception event into its own kel. + # Otherwise failed make raises exception ConfigurationError + assert wopHab.pre == 'BIDO3FhB5smF6WsTJkRRAao_wEttcbsBnDCmfQ4_f1b_' + wopRemKvy = eventing.Kevery(db=wopHby.db, lax=False, local=False) # for remote events + + topHab = topHby.makeHab(name="top", icount=1, isith='1', # single sig + ncount=1, nsith='1', # single next + wits=[wopHab.pre], toad=1) # one witness + # makehab also enters inception event into its own kel + # Otherwise failed make raises exception ConfigurationError + assert topHab.pre == 'EJcCaHg3AtW_gRzpaz6Pw03Yv49is2IJDRwYE7ey91KE' + topRemKvy = eventing.Kevery(db=topHby.db, lax=False, local=False) # for remote events + + # be witness to controller's inception + # first make inception + stream = topHab.makeOwnInception() + assert stream == (b'{"v":"KERI10JSON000159_","t":"icp","d":"EJcCaHg3AtW_gRzpaz6Pw03Y' + b'v49is2IJDRwYE7ey91KE","i":"EJcCaHg3AtW_gRzpaz6Pw03Yv49is2IJDRwYE' + b'7ey91KE","s":"0","kt":"1","k":["DPJVPYS9efLUHDOqxwG6pxISZSRACgNf' + b'uZm7qK7DzQKD"],"nt":"1","n":["EN8AnwKGnCOAyP2FRXuQMyMjbRsDjRpDd_' + b'_ZK2KuL0ID"],"bt":"1","b":["BIDO3FhB5smF6WsTJkRRAao_wEttcbsBnDCm' + b'fQ4_f1b_"],"c":[],"a":[]}-AABAADPMCL6P4DYi3qvgR4v1UcrCYMjnmRx-xJ' + b'meOfA8b8gdHZxZvVgpgLrAxFYwSEAtdhGT8LOPdGpTqxWSocCmXkH') + + # add test fail process as remote since since witness of controller + + # first process as local since witness + wopHab.psr.parse(ims=stream) # now have controller's inception in db + assert topHab.pre in wopHab.kevers # success + + serder = wopHab.kevers[topHab.pre].serder + # generate witness receipt and process + receipt = wopHab.witness(serder=serder) # now has fully witnessd controller icp + count = wopHab.db.cntWigs(dbing.dgKey(topHab.pre, serder.said)) + assert count >= 1 + + assert receipt == (b'{"v":"KERI10JSON000091_","t":"rct","d":"EJcCaHg3AtW_gRzpaz6Pw03Y' + b'v49is2IJDRwYE7ey91KE","i":"EJcCaHg3AtW_gRzpaz6Pw03Yv49is2IJDRwYE' + b'7ey91KE","s":"0"}-VAX-BABAACf4sllk4USRirj3xNlnFgDcbWHsAi6kOigNHr' + b'Ddbde06NhDELtWTeJOcz7T_rru_rpd6Uov4IN_0rthtMbxgcI') + + # add test fail process as remote since own witness + + # process receipt as local since own witness receipt. + topHab.psr.parse(ims=receipt) # now top has fully witnessed icp. + count = topHab.db.cntWigs(dbing.dgKey(topHab.pre, serder.said)) + assert count >= 1 + + # Create witness wid and delegated controller mid + widHab = widHby.makeHab(name="wid", transferable=False) # witness nontrans + # makehab also enters inception event into its own kel. + # Otherwise failed make raises exception ConfigurationError + assert widHab.pre == 'BCI95exU-RepxQ0HmGcp7USLMCPxrXKzMc1DXqfRnikP' + widRemKvy = eventing.Kevery(db=widHby.db, lax=False, local=False) # for remote events + + midHab = midHby.makeHab(name="mid", icount=1, isith='1', # single sig + ncount=1, nsith='1', # single next + wits=[widHab.pre], toad=1, # one witness + delpre=topHab.pre) # delegated + # makehab also enters inception event into its own kel + # Otherwise failed make raises exception ConfigurationError + assert midHab.pre == 'EEaTQhI7QGM-usOJtpKM9L0yQjGiBYJC3tq905aC8am4' + assert midHab.delpre == topHab.pre + midRemKvy = eventing.Kevery(db=midHby.db, lax=False, local=False) # for remote events + + # be witness to controller's inception. + # first make inception + stream = midHab.makeOwnInception() + + # add test fail process as remote since since witness of controller + + #first process as local since witness + widHab.psr.parse(ims=stream) # now have controller's inception in db + assert midHab.pre in widHab.kevers # success + + serder = widHab.kevers[midHab.pre].serder + # generate witness receipt and process + receipt = widHab.witness(serder=serder) # now has fully witnessed controller icp + count = widHab.db.cntWigs(dbing.dgKey(midHab.pre, serder.said)) + assert count >= 1 + + # add test fail process as remote since own witness + + # top process wop receipt as local since own witness receipt. + midHab.psr.parse(ims=receipt) # now top has fully witnessed icp. + count = midHab.db.cntWigs(dbing.dgKey(midHab.pre, serder.said)) + assert count >= 1 + + # Create witness wot and controller bot + wotHab = widHby.makeHab(name="wot", transferable=False) # witness nontrans + # makehab also enters inception event into its own kel. + # Otherwise failed make raises exception ConfigurationError + assert wotHab.pre == 'BDChA_O6twrlHcXKf7xu1xYee__nZxDbBa0W_XznpLQH' + wotRemKvy = eventing.Kevery(db=wotHby.db, lax=False, local=False) # for remote events + + botHab = botHby.makeHab(name="bot", icount=1, isith='1', # single sig + ncount=1, nsith='1', # single next + wits=[wotHab.pre], toad=1, # one witness + delpre=midHab.pre) # delegated + # makehab also enters inception event into its own kel + # Otherwise failed make raises exception ConfigurationError + assert botHab.pre == 'EPtHqQJwlEj2sM0e2WslvwSsAsxAflmn7JIabs-LBqJC' + assert botHab.delpre == midHab.pre + botRemKvy = eventing.Kevery(db=botHby.db, lax=False, local=False) # for remote events + + # be witness to controller's inception. + # first make inception + stream = botHab.makeOwnInception() + + # add test fail process as remote since since witness of controller + + #first process as local since witness + wotHab.psr.parse(ims=stream) # now have controller inception in db + assert botHab.pre in wotHab.kevers # success + + serder = wotHab.kevers[botHab.pre].serder + # generate witness receipt and process + receipt = wotHab.witness(serder=serder) # now has fully witnessed controller icp + count = wotHab.db.cntWigs(dbing.dgKey(botHab.pre, serder.said)) + assert count >= 1 + + # add test fail process as remote since own witness + + # top process wop receipt as local since own witness receipt. + botHab.psr.parse(ims=receipt) # now top has fully witnessed icp. + count = botHab.db.cntWigs(dbing.dgKey(botHab.pre, serder.said)) + assert count >= 1 + + # This needs to be fixedup to actually test delegating superseding recovery + # TODO? + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS): + + # Init key pair managers + bobMgr = keeping.Manager(ks=bobKS, salt=topSalt) + delMgr = keeping.Manager(ks=delKS, salt=midSalt) + + # Init Keverys + bobKvy = eventing.Kevery(db=bobDB) + delKvy = eventing.Kevery(db=delDB) + + # Setup Bob by creating inception event + verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted + bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + bob = bobSrdr.ked["i"] + assert bob == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + + bobMgr.move(old=verfers[0].qb64, new=bob) # move key pair label to prefix + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) + + msg = bytearray(bobSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + assert msg == (b'{"v":"KERI10JSON00012b_","t":"icp","d":"EA_SbBUZYwqLVlAAn14d6QUB' + b'QCSReJlZ755JqTgmRhXH","i":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755Jq' + b'TgmRhXH","s":"0","kt":"1","k":["DKiNnDmdOkcBjcAqL2FFhMZnSlPfNyGr' + b'JlCjJmX5b1nU"],"nt":"1","n":["EMP7Lg6BtehOYZt2RwOqXLNfMUiUllejAp' + b'8G_5EiANXR"],"bt":"0","b":[],"c":[],"a":[]}-AABAAArkDBeflIAo4kBs' + b'Knc754XHJvdLnf04iq-noTFEJkbv2MeIGZtx6lIfJPmRSEmFMUkFW4otRrMeBGQ0' + b'-nlhHEE') + + # apply msg to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + bobK = bobKvy.kevers[bob] + assert bobK.prefixer.qb64 == bob + assert bobK.serder.said == bobSrdr.said + assert bobK.serder.said == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert bob in delKvy.kevers + + # Setup Del's inception event assuming that Bob's next event will be an ixn delegating event + verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted + + delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], + delpre=bobK.prefixer.qb64, + ndigs=[diger.qb64 for diger in digers]) + + delPre = delSrdr.ked["i"] + assert delPre == 'EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj' + + delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + assert delSrdr.said == 'EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj' + + # Now create delegating event + seal = eventing.SealEvent(i=delPre, + s=delSrdr.ked["s"], + d=delSrdr.said) + bobSrdr = eventing.interact(pre=bobK.prefixer.qb64, + dig=bobK.serder.said, + sn=bobK.sn + 1, + data=[seal._asdict()]) + + assert bobSrdr.said == 'EJtQndkvwnMpVGE5oVVbLWSCm-jLviGw1AOOkzBvNwsS' + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) + msg = bytearray(bobSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + assert msg == (b'{"v":"KERI10JSON00013a_","t":"ixn","d":"EJtQndkvwnMpVGE5oVVbLWSC' + b'm-jLviGw1AOOkzBvNwsS","i":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755Jq' + b'TgmRhXH","s":"1","p":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRh' + b'XH","a":[{"i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj","s"' + b':"0","d":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj"}]}-AABAA' + b'DFmoctrQkBbm47vuk7ejMbQ1y5vKD0Nfo8cqzbETZAlEPdbgVRSFta1-Bpv0y1Ri' + b'DrCxa_0IOp906gYqDPXIwG') + + # apply msg to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert delKvy.kevers[bob].serder.said == bobSrdr.said + + # now create msg with Del's delegated inception event + sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) + + msg = bytearray(delSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + seqner = coring.Seqner(sn=bobK.sn) + msg.extend(seqner.qb64b) + msg.extend(bobSrdr.saidb) + + assert msg == (b'{"v":"KERI10JSON00015f_","t":"dip","d":"EHng2fV42DdKb5TLMIs6bbjF' + b'kPNmIdQ5mSFn6BTnySJj","i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6' + b'BTnySJj","s":"0","kt":"1","k":["DLitcfMnabnLt-PNCaXdVwX45wsG93Wd' + b'8eW9QiZrlKYQ"],"nt":"1","n":["EDjXvWdaNJx7pAIr72Va6JhHxc7Pf4ScYJ' + b'G496ky8lK8"],"bt":"0","b":[],"c":[],"a":[],"di":"EA_SbBUZYwqLVlA' + b'An14d6QUBQCSReJlZ755JqTgmRhXH"}-AABAABv6Q3s-1Tif-ksrx7ul9OKyOL_Z' + b'PHHp6lB9He4n6kswjm9VvHXzWB3O7RS2OQNWhx8bd3ycg9bWRPRrcKADoYC-GAB0' + b'AAAAAAAAAAAAAAAAAAAAAABEJtQndkvwnMpVGE5oVVbLWSCm-jLviGw1AOOkzBvN' + b'wsS') + + + # apply Del's delegated inception event message to Del's own Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert delPre in delKvy.kevers + delK = delKvy.kevers[delPre] + assert delK.delegated + assert delK.serder.said == delSrdr.said + couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # apply Del's delegated inception event message to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert delPre in bobKvy.kevers # successfully validated + bobDelK = bobKvy.kevers[delPre] + assert bobDelK.delegated + assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # Setup Del rotation event assuming that Bob's next event will be an ixn delegating event + verfers, digers = delMgr.rotate(pre=delPre, temp=True) + + delSrdr = eventing.deltate(pre=bobDelK.prefixer.qb64, + keys=[verfer.qb64 for verfer in verfers], + dig=bobDelK.serder.said, + sn=bobDelK.sn + 1, + ndigs=[diger.qb64 for diger in digers]) + + assert delSrdr.said == 'EM5fj7YtOQYH3iLyWJr6HZVVxrY5t46LRL2vkNpdnPi0' + + # Now create delegating interaction event + seal = eventing.SealEvent(i=bobDelK.prefixer.qb64, + s=delSrdr.ked["s"], + d=delSrdr.said) + bobSrdr = eventing.interact(pre=bobK.prefixer.qb64, + dig=bobK.serder.said, + sn=bobK.sn + 1, + data=[seal._asdict()]) + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) + + msg = bytearray(bobSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + assert msg == (b'{"v":"KERI10JSON00013a_","t":"ixn","d":"EJaPTWDiWvay8voiJkbxkvoa' + b'buUf_1a22yk9tVdRiMVs","i":"EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755Jq' + b'TgmRhXH","s":"2","p":"EJtQndkvwnMpVGE5oVVbLWSCm-jLviGw1AOOkzBvNw' + b'sS","a":[{"i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnySJj","s"' + b':"1","d":"EM5fj7YtOQYH3iLyWJr6HZVVxrY5t46LRL2vkNpdnPi0"}]}-AABAA' + b'C8htl4epY7F5QBjro00VdfisxZMZWRXfe6xX_nVfS5gOsv8HOkzUKYMsvAVG4TJg' + b'7n1u44IyfsiKrB2R_UeUIK') + + # apply msg to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert delKvy.kevers[bob].serder.said == bobSrdr.said + + # now create msg from Del's delegated rotation event + sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) + + msg = bytearray(delSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + counter = core.Counter(core.Codens.SealSourceCouples, count=1, + gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + seqner = coring.Seqner(sn=bobK.sn) + msg.extend(seqner.qb64b) + msg.extend(bobSrdr.saidb) + + assert msg ==(b'{"v":"KERI10JSON000160_","t":"drt","d":"EM5fj7YtOQYH3iLyWJr6HZVV' + b'xrY5t46LRL2vkNpdnPi0","i":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6' + b'BTnySJj","s":"1","p":"EHng2fV42DdKb5TLMIs6bbjFkPNmIdQ5mSFn6BTnyS' + b'Jj","kt":"1","k":["DE3-kGVqHrdeeKPcL83jLjYS0Ea_CWgFHogusIwf-P9P"' + b'],"nt":"1","n":["EMj2mWvNvn6w9BbGUADX1AU3vn7idcUffZIaCvAsibru"],' + b'"bt":"0","br":[],"ba":[],"a":[]}-AABAAB_x-9_FTWr-OW_xXBN5pUkFNqL' + b'pAqTTQC02sPysnP0WmBFHb8NWvog9F-o279AfpPcLMxktypg1Fz7EQFYCuwC-GAB' + b'0AAAAAAAAAAAAAAAAAAAAAACEJaPTWDiWvay8voiJkbxkvoabuUf_1a22yk9tVdR' + b'iMVs') + + # apply msg to del's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=delKvy) + # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + assert bobDelK.delegated + assert delK.serder.said == delSrdr.said + couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # apply Del's delegated inception event message to bob's Kevery + parsing.Parser().parse(ims=bytearray(msg), kvy=bobKvy) + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert bobDelK.delegated + assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # test replay + msgs = bytearray() + for msg in delKvy.db.clonePreIter(pre=delPre, fn=0): + msgs.extend(msg) + assert len(msgs) == 1167 + assert couple in msgs + + assert not os.path.exists(delKS.path) + assert not os.path.exists(delDB.path) + assert not os.path.exists(bobKS.path) + assert not os.path.exists(bobDB.path) + + """End Test""" + + +def test_delegables_escrow(): + gateSalt = core.Salter(raw=b'0123456789abcdef').qb64 + torSalt = core.Salter(raw=b'0123456789defabc').raw + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + with habbing.openHby(name="delegate", temp=True, salt=gateSalt) as gateHby, \ + habbing.openHab(name="delegator", temp=True, salt=torSalt) as (torHby, torHab): + + gateHab = gateHby.makeHab(name="repTest", transferable=True, delpre=torHab.pre) + assert gateHab.pre == "EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd" + + gateIcp = gateHab.makeOwnEvent(sn=0) + torKvy = eventing.Kevery(db=torHab.db, lax=False, local=False) + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) + assert gateHab.pre not in torKvy.kevers + assert len(torHab.db.delegables.get(keys=snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn))) == 1 + # Exercise the MissingDelegableApprovalError case + torKvy.processEscrowDelegables() + + # Now create delegating interaction event + seal = eventing.SealEvent(i=gateHab.pre, + s="0", + d=gateHab.pre) + ixn = torHab.interact(data=[seal._asdict()]) + assert ixn == (b'{"v":"KERI10JSON00013a_","t":"ixn","d":"EPUCIjCibL-VeT3n6PYIkbyP' + b'qpioIFT79NRqxboFv0Os","i":"EJTtW40aDl0aKDZ09v-o6uDz_VwLJGplp6WTI' + b'BGCoVog","s":"1","p":"EJTtW40aDl0aKDZ09v-o6uDz_VwLJGplp6WTIBGCoV' + b'og","a":[{"i":"EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd","s"' + b':"0","d":"EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd"}]}-AABAA' + b'BRR9HDRx_7KdWJ7uokLzREP3c1Hg7Grq5fwoGl_EXA-reR05aYPjDdZ4CIZTnqDo' + b'EN2hqNbHfq4zMaDlR8Ja4D') + + # Make sure that our anchoring ixn event is in our own KEL + assert torHab.kever.sn == 1 + + # Place the anchor seal in the database... this will be retrieved from the fully committed delegate event + serder = torHab.kever.serder + seqner = coring.Seqner(sn=serder.sn) + couple = seqner.qb64b + serder.saidb + dgkey = dbing.dgKey(gateHab.kever.prefixer.qb64b, gateHab.kever.serder.saidb) + torHab.db.setAes(dgkey, couple) # authorizer event seal (delegator/issuer) + + # delegate still not kevers + assert gateHab.pre not in torKvy.kevers + assert len(torHab.db.delegables.get(keys=snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn))) == 1 + + # run the delegables escrow processor to make get delegate in our Kevers + torKvy.processEscrowDelegables() + assert len(torHab.db.delegables.get(keys=snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn))) == 0 + assert gateHab.pre in torKvy.kevers + if __name__ == "__main__": test_delegation() + test_delegation_supersede() + test_delegables_escrow() + diff --git a/tests/core/test_escrow.py b/tests/core/test_escrow.py index cac27e311..decdbc01b 100644 --- a/tests/core/test_escrow.py +++ b/tests/core/test_escrow.py @@ -3,15 +3,23 @@ tests escrows in database primarily logic in Kevery and Kever from keri.core.eventing """ +import logging import os import time import datetime +import pytest + from keri import help +from keri.core.eventing import deWitnessCouple, deReceiptTriple, deTransReceiptQuintuple +from keri.db.dbing import splitSnKey, dgKey, snKey from keri.help import helping + +from keri import core, kering +from keri.core import coring, eventing, parsing, indexing, serdering + from keri.db import dbing, basing -from keri.app import keeping -from keri.core import coring, eventing, parsing +from keri.app import keeping, habbing logger = help.ogler.getLogger() @@ -21,8 +29,9 @@ def test_partial_signed_escrow(): Test partially signed escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -51,7 +60,8 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) @@ -81,7 +91,8 @@ def test_partial_signed_escrow(): # Send message again but with signature from other siger msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[2].qb64b) # apply msg to Kevery to process @@ -115,8 +126,8 @@ def test_partial_signed_escrow(): # send duplicate message with all three sigs msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -144,7 +155,7 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=kvr.verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[1].qb64b) @@ -158,7 +169,7 @@ def test_partial_signed_escrow(): # add another sig msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) @@ -186,7 +197,7 @@ def test_partial_signed_escrow(): # resend events to load escrow msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[1].qb64b) @@ -200,7 +211,7 @@ def test_partial_signed_escrow(): # add another sig msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) @@ -229,7 +240,7 @@ def test_partial_signed_escrow(): # send duplicate message but add last sig msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[2].qb64b) psr.parse(ims=bytearray(msg), kvy=kvy) @@ -266,8 +277,8 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -296,7 +307,8 @@ def test_partial_signed_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=2) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=2, + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[0].qb64b) msg.extend(sigers[3].qb64b) @@ -311,7 +323,7 @@ def test_partial_signed_escrow(): assert kvr.serder.said != srdr.said # key state not updated msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) msg.extend(sigers[1].qb64b) @@ -342,45 +354,179 @@ def test_partial_signed_escrow(): """End Test""" +def test_partial_signed_escrow_validation_errors(): + """ + Test partially signed escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + # init event DB and keep DB + with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Identifier setup - 3 signers so we can easily trigger the partially signed state + # create inception event with 3 keys each in incept and next sets + # defaults are algo salty and rooted + sith = ["1/2", "1/2", "1/2"] # 2 of 3 but with weighted threshold + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='wes', temp=True) + srdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + isith=sith, + nsith=nxtsith, + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + pre = srdr.ked["i"] + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + + # Create CESR message byte stream with attachments + msg = bytearray(srdr.raw) # message + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + counter = core.Counter(core.Codens.ControllerIdxSigs, gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # attachments + msg.extend(sigers[0].qb64b) # only add the first signer to trigger partially signed logic + + # apply msg to Kevery to process + psr.parse(ims=bytearray(msg), kvy=kvy) + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + assert escrows[0] == srdr.saidb # escrow entry for event + + # TEST MISSING EVENT SOURCE + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + esr = db.esrs.get(keys=dgkey) + assert esr is not None + db.esrs.rem(keys=dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows)== 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(msg), kvy=kvy) + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + dts = db.getDts(dgkey) + assert dts is not None + db.delDts(dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(msg), kvy=kvy) + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + evt = db.getEvt(dgkey) + assert evt is not None + db.delEvt(dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(msg), kvy=kvy) + escrows = kvy.db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 1 + for ekey, edig in db.getPseItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + sigs = db.getSigs(dgkey) + assert sigs is not None + db.delSigs(dgkey) + kvy.processEscrowPartialSigs() + escrows = db.getPses(dbing.snKey(pre, int(srdr.ked["s"], 16))) + assert len(escrows) == 0 + + def test_missing_delegator_escrow(): """ Test missing delegator escrow + + bob is the delegator + del is the delegate + wat is the watcher """ - # bob is the delegator del is bob's delegate - bobSalt = coring.Salter(raw=b'0123456789abcdef').qb64 - delSalt = coring.Salter(raw=b'abcdef0123456789').qb64 + bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 + delSalt = core.Salter(raw=b'abcdef0123456789').qb64 + watSalt = core.Salter(raw=b'wxyzabcdefghijkl').qb64 psr = parsing.Parser() - with basing.openDB(name="bob") as bobDB, \ - keeping.openKS(name="bob") as bobKS, \ - basing.openDB(name="del") as delDB, \ - keeping.openKS(name="del") as delKS: + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS, + basing.openDB(name="wat") as watDB, + keeping.openKS(name="wat") as watKS): # Init key pair managers bobMgr = keeping.Manager(ks=bobKS, salt=bobSalt) delMgr = keeping.Manager(ks=delKS, salt=delSalt) + watMgr = keeping.Manager(ks=watKS, salt=watSalt) # Init Keverys bobKvy = eventing.Kevery(db=bobDB) delKvy = eventing.Kevery(db=delDB) + watKvy = eventing.Kevery(db=watDB) + + # Setup Wat with own inception event + verfers, digers = watMgr.incept(stem='wat', temp=True) # algo default salty and rooted + + watSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + watPre = watSrdr.pre + watMgr.move(old=verfers[0].qb64, new=watPre) # move key pair label to prefix + # Setup wat's prefixes so wat's KEL will be Kever.locallyOwned() + watDB.prefixes.add(watPre) + assert watPre in watDB.prefixes + # setup wat's on kel + sigers = watMgr.sign(ser=watSrdr.raw, verfers=verfers) + msg = bytearray(watSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + watIcpMsg = msg # save for later - # Setup Bob by creating inception event + # apply msg to wats's Kevery + psr.parse(ims=bytearray(watIcpMsg), kvy=watKvy, local=True) + watK = watKvy.kevers[watPre] + assert watK.prefixer.qb64 == watPre + assert watK.serder.said == watSrdr.said + + # Setup Bob with own inception event verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], ndigs=[diger.qb64 for diger in digers], code=coring.MtrDex.Blake3_256) - bobPre = bobSrdr.ked["i"] - + bobPre = bobSrdr.pre bobMgr.move(old=verfers[0].qb64, new=bobPre) # move key pair label to prefix + # Setup Bob's prefixes so bob's KEL will be Kever.locallyOwned() and + # Del's KEL will be Kever.locallyDelegated() + bobDB.prefixes.add(bobPre) + assert bobPre in bobDB.prefixes sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) - msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -388,24 +534,32 @@ def test_missing_delegator_escrow(): bobIcpMsg = msg # save for later # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIcpMsg), kvy=bobKvy, local=True) bobK = bobKvy.kevers[bobPre] assert bobK.prefixer.qb64 == bobPre assert bobK.serder.said == bobSrdr.said + assert bobK.sn == 0 - # Setup Del's inception event assuming that Bob's next event will be an ixn delegating event - verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted + # apply msg to del's Kevery so he knows about the AID + psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) + assert bobK.prefixer.qb64 in delKvy.kevers + delBobK = bobKvy.kevers[bobPre] # bobs kever in dels kevery + assert delBobK.sn == 0 + # Setup Del's inception event assuming that Bob's next event will be + # an ixn delegating event + verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], delpre=bobPre, ndigs=[diger.qb64 for diger in digers]) - delPre = delSrdr.ked["i"] - + delPre = delSrdr.pre delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + # Setup Del's prefixes so Del's KEL will be Kever.locallyOwned() + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes - # Now create delegating event + # Now create delegating event for Bob seal = eventing.SealEvent(i=delPre, s=delSrdr.ked["s"], d=delSrdr.said) @@ -417,36 +571,39 @@ def test_missing_delegator_escrow(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - bobIxnMsg = msg + bobIxnMsg1 = msg # delegating event with attachments # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(bobIxnMsg1), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 1 - # now create msg with Del's delegated inception event + # now create Del's delegated inception event msg sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delIcpMsg = msg # apply Del's delegated inception event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) + # because the attachment includes valid source seal then the Delegables + # escrow is bypassed and is validated and shows up in AES + psr.parse(ims=bytearray(delIcpMsg), kvy=bobKvy, local=True) # bobKvy.process(ims=bytearray(msg)) # process local copy of msg assert delPre in bobKvy.kevers # successfully validated bobDelK = bobKvy.kevers[delPre] # delK in bobs kevery @@ -456,55 +613,50 @@ def test_missing_delegator_escrow(): assert couple == seqner.qb64b + bobSrdr.saidb # apply Del's inception msg to Del's Kevery - # Dels event will fail but will add to its escrow - psr.parse(ims=bytearray(msg), kvy=delKvy) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delPre not in delKvy.kevers - assert bobPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) - assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - - # verify Kevery process partials escrow is idempotent to previously escrowed events - # assuming not stale but nothing else has changed - delKvy.processEscrowPartialSigs() - assert delPre not in delKvy.kevers - assert bobPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + # Because locallyOwned by delegate event does not validate delegation + # and ignores the attached source seal + psr.parse(ims=bytearray(delIcpMsg), kvy=delKvy, local=True) + assert delPre in delKvy.kevers + delK = delKvy.kevers[delPre] + # no AES entry for del's own delegated event when locallyOwned + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + + # apply Del's delegated inception event message to wats's Kevery as remote + # because the attachment includes valid source seal but wat does not + # yet have Bob's delegating event entry. The event goes into partial + # delegated event escrow + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - - # apply Bob's inception to Dels' Kvy - psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy) - # delKvy.process(ims=bytearray(bobIcpMsg)) # process remote copy of msg - assert bobPre in delKvy.kevers # message accepted - delKvy.processEscrowPartialSigs() # process escrow - assert delPre not in delKvy.kevers - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + assert escrows[0] == delSrdr.said # escrow entry for event + + # Now apply Bob's incept to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIcpMsg), kvy=watKvy, local=False) + assert bobPre in watKvy.kevers + watBobK = watKvy.kevers[bobPre] + assert watBobK.sn == 0 + watKvy.processEscrows() + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 1 - assert escrows[0] == delSrdr.saidb # escrow entry for event - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow == seqner.qb64b + bobSrdr.saidb # escrow entry for event - - # apply Bob's delegating interaction to Dels' Kvy - psr.parse(ims=bytearray(bobIxnMsg), kvy=delKvy) - # delKvy.process(ims=bytearray(bobIxnMsg)) # process remote copy of msg - delKvy.processEscrowPartialSigs() # process escrows - assert delPre in delKvy.kevers # event removed from escrow - delK = delKvy.kevers[delPre] - assert delK.delegated - assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - assert couple == seqner.qb64b + bobSrdr.saidb + assert escrows[0] == delSrdr.said # escrow entry for event - escrows = delKvy.db.getPses(dbing.snKey(delPre, int(delSrdr.ked["s"], 16))) + # Now apply Bob's ixn to wat's kvy and process escrow + psr.parse(ims=bytearray(bobIxnMsg1), kvy=watKvy, local=False) + watKvy.processEscrows() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) assert len(escrows) == 0 - escrow = delKvy.db.getPde(dbing.dgKey(delPre, delSrdr.said)) - assert escrow is None # delegated inception delegation couple + assert watBobK.sn == 1 + + assert delPre in watKvy.kevers # successfully validated + watDelK = watKvy.kevers[delPre] # delK in wats kevery + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + # Setup Del rotation event verfers, digers = delMgr.rotate(pre=delPre, temp=True) @@ -527,54 +679,66 @@ def test_missing_delegator_escrow(): sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) msg = bytearray(bobSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - # apply msg to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + bobIxnMsg2 = msg + + # apply bobs IXN msg to bob's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=bobKvy, local=True) assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 2 # apply msg to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg - assert delKvy.kevers[bobPre].serder.said == bobSrdr.said + psr.parse(ims=bytearray(bobIxnMsg2), kvy=delKvy, local=True) + assert delBobK.serder.said == bobSrdr.said + assert delBobK.sn == 2 + + # apply msg to wat's Kevery + psr.parse(ims=bytearray(bobIxnMsg2), kvy=watKvy, local=True) + assert watBobK.serder.said == bobSrdr.said + assert watBobK.sn == 2 # now create msg from Del's delegated rotation event sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) - msg = bytearray(delSrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) seqner = coring.Seqner(sn=bobK.sn) msg.extend(seqner.qb64b) msg.extend(bobSrdr.saidb) + delRotMsg = msg + # apply Del's delegated Rotation event message to del's Kevery - psr.parse(ims=bytearray(msg), kvy=delKvy) - # delKvy.process(ims=bytearray(msg)) # process remote copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=delKvy, local=True) assert delK.delegated assert delK.serder.said == delSrdr.said - couple = delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) - assert couple == seqner.qb64b + bobSrdr.saidb + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) # apply Del's delegated Rotation event message to bob's Kevery - psr.parse(ims=bytearray(msg), kvy=bobKvy) - # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + psr.parse(ims=bytearray(delRotMsg), kvy=bobKvy, local=True) assert bobDelK.delegated assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) assert couple == seqner.qb64b + bobSrdr.saidb + # apply Del's delegated Rotation event message to wats's Kevery + psr.parse(ims=bytearray(delRotMsg), kvy=watKvy, local=True) + assert watDelK.delegated + assert watDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = watKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + assert not os.path.exists(delKS.path) @@ -585,13 +749,26 @@ def test_missing_delegator_escrow(): """End Test""" +def test_misfit_escrow(): + """ + Test misfit escrow + + """ + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + + # stub for now + + """End Test""" + + def test_out_of_order_escrow(): """ Test out of order escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -621,8 +798,8 @@ def test_out_of_order_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -635,8 +812,8 @@ def test_out_of_order_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -664,8 +841,8 @@ def test_out_of_order_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -781,13 +958,140 @@ def test_out_of_order_escrow(): """End Test""" +def test_out_of_order_escrow_validation_errors(): + """ + Test out-of-order escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + # init event DB and keep DB + with basing.openDB(name="dandy") as db, keeping.openKS(name="dandy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Identifier setup + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='wes', temp=True) + + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + + # Create CESR message byte stream with attachments + msg = bytearray(srdr.raw) # message + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # Attachments + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = bytearray(msg) # save copy for later + + # create interaction event + srdr = eventing.interact(pre=pre, dig=icpdig, sn=1, data=[]) + ixndig = srdr.said + + # Create CESR message byte stream with attachments + msg = bytearray(srdr.raw) # message + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # attachments + for siger in sigers: + msg.extend(siger.qb64b) + ixnmsg = bytearray(msg) # save copy for later + + # TEST MISSING EVENT SOURCE + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + esr = db.esrs.get(keys=dgkey) + assert esr is not None + db.esrs.rem(keys=dgkey) + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # re-apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + dtb = db.getDts(dgkey) + assert dtb is not None + db.delDts(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # stale event escrow test covered by test_out_of_order_escrow above + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # re-apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + eraw = db.getEvt(dgkey) + assert eraw is not None + db.delEvt(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) # re-apply ixn msg, adding to OOO escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + assert escrows[0] == ixndig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getOoeItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + sigs = db.getSigs(dgkey) + assert sigs is not None + db.delSigs(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowOutOfOrders() # will remove the now invalid escrowed interaction event + escrows = db.getOoes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # test high level processEscrows exception handler + del db.sigs # intentionally breaks the Baser db instance so that an unhandled exception occurs + with pytest.raises(Exception): + kvy.processEscrows() # will raise exception caught by the Kevery.processEscrows catch all + # this is to increase test coverage + + def test_unverified_receipt_escrow(): """ Test unverified receipt escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -831,8 +1135,8 @@ def test_unverified_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -845,7 +1149,8 @@ def test_unverified_receipt_escrow(): wit0Cigar = mgr.sign(ser=srdr.raw, verfers=[wit0Verfer], indexed=False)[0] # returns Cigar unindexed wit1Cigar = mgr.sign(ser=srdr.raw, verfers=[wit1Verfer], indexed=False)[0] # returns Cigar unindexed - recnt = coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=2) + recnt = core.Counter(core.Codens.NonTransReceiptCouples, count=2, + gvrsn=kering.Vrsn_1_0) msg = bytearray() msg.extend(reserder.raw) @@ -878,8 +1183,8 @@ def test_unverified_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -892,7 +1197,8 @@ def test_unverified_receipt_escrow(): wit0Cigar = mgr.sign(ser=srdr.raw, verfers=[wit0Verfer], indexed=False)[0] # returns Cigar unindexed wit1Cigar = mgr.sign(ser=srdr.raw, verfers=[wit1Verfer], indexed=False)[0] # returns Cigar unindexed - recnt = coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=2) + recnt = core.Counter(core.Codens.NonTransReceiptCouples, count=2, + gvrsn=kering.Vrsn_1_0) msg = bytearray() msg.extend(reserder.raw) @@ -940,8 +1246,8 @@ def test_unverified_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -954,7 +1260,8 @@ def test_unverified_receipt_escrow(): wit0Cigar = mgr.sign(ser=srdr.raw, verfers=[wit0Verfer], indexed=False)[0] # returns Cigar unindexed wit1Cigar = mgr.sign(ser=srdr.raw, verfers=[wit1Verfer], indexed=False)[0] # returns Cigar unindexed - recnt = coring.Counter(code=coring.CtrDex.NonTransReceiptCouples, count=2) + recnt = core.Counter(core.Codens.NonTransReceiptCouples, count=2, + gvrsn=kering.Vrsn_1_0) msg = bytearray() msg.extend(reserder.raw) @@ -1071,8 +1378,9 @@ def test_unverified_trans_receipt_escrow(): Test unverified transferable receipt escrow """ - salt = coring.Salter(raw=b'0123456789abcdef').qb64 # init Salter + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks # init event DB and keep DB with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: @@ -1103,8 +1411,8 @@ def test_unverified_trans_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -1130,8 +1438,8 @@ def test_unverified_trans_receipt_escrow(): rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) msg = bytearray(rsrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(rsigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(rsigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in rsigers: msg.extend(siger.qb64b) @@ -1170,8 +1478,8 @@ def test_unverified_trans_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -1196,8 +1504,8 @@ def test_unverified_trans_receipt_escrow(): rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) msg = bytearray(rsrdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(rsigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(rsigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in rsigers: msg.extend(siger.qb64b) @@ -1250,8 +1558,8 @@ def test_unverified_trans_receipt_escrow(): sigers = mgr.sign(ser=srdr.raw, verfers=verfers) msg = bytearray(srdr.raw) - counter = coring.Counter(code=coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -1399,6 +1707,1409 @@ def test_unverified_trans_receipt_escrow(): """End Test""" +def test_partial_wigs_validation_errors(): + """ + Test partial witness receipt escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + # init event DB and keep DB + with basing.openDB(name="fredy") as db, keeping.openKS(name="fredy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # create witness identifiers + verfers, digers = mgr.incept(ncount=0, stem="wit0", transferable=False, temp=True) + wit0Verfer = verfers[0] + wit0pre = wit0Verfer.qb64 + + verfers, digers = mgr.incept(ncount=0, stem="wit1", transferable=False, temp=True) + wit1Verfer = verfers[0] + wit1pre = wit1Verfer.qb64 + + assert wit1pre != wit0pre + assert wit1pre < wit0pre # means wit1 escrow will get serviced first + + # create inception event with 3 keys each in incept and next sets + # defaults are algo salty and rooted + sith = ["1/2", "1/2", "1/2"] # 2 of 3 but with weighted threshold + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='fredy', temp=True) + + srdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + isith=sith, + nsith=nxtsith, + wits=[wit0pre, wit1pre], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + pre = srdr.ked["i"] + icpdig = srdr.said + + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # Create inception CESR message and attachments byte stream + msg = bytearray(srdr.raw) # message + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # attachments + for siger in sigers: + msg.extend(siger.qb64b) + + icpmsg = msg + + + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + # First run of escrow processing should loop and not remove event + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + # TEST MISSING EVENT SOURCE + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + esr = db.esrs.get(keys=dgkey) + assert esr is not None + db.esrs.rem(keys=dgkey) # Removes event source + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + dtb = db.getDts(dgkey) + assert dtb is not None + db.delDts(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + eraw = db.getEvt(dgkey) + assert eraw is not None + db.delEvt(dgkey) # remove the datetime stamp of the event so it will be invalid + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed interaction event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + for ekey, edig in db.getPweItemIter(): + pre, sn = splitSnKey(ekey) + dgkey = dgKey(pre, bytes(edig)) + sigers = db.getSigs(dgkey) + assert sigers is not None + db.delSigs(dgkey) + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed interaction event + escrows = db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply icp msg, adding to PW escrow + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 1 + assert escrows[0] == icpdig.encode("utf-8") # escrow entry for event + + kvy.TimeoutPWE = 0 # forces all escrows to be stale + time.sleep(0.001) + kvy.processEscrowPartialWigs() # will remove the now invalid escrowed inception event + assert pre not in kvy.kevers # event not accepted + escrows = kvy.db.getPwes(dbing.snKey(pre, 0)) + assert len(escrows) == 0 + + +def test_partial_delegation_escrow_validation_errors(): + """ + Test partial delegation escrow validation errors which each should remove the invalid escrowed events + and not continue to reprocess them. + """ + bobSalt = core.Salter(raw=b'0123456789abcdef').qb64 + delSalt = core.Salter(raw=b'abcdef0123456789').qb64 + watSalt = core.Salter(raw=b'wxyzabcdefghijkl').qb64 + + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + psr = parsing.Parser() + + with (basing.openDB(name="bob") as bobDB, + keeping.openKS(name="bob") as bobKS, + basing.openDB(name="del") as delDB, + keeping.openKS(name="del") as delKS, + basing.openDB(name="wat") as watDB, + keeping.openKS(name="wat") as watKS): + + # Init key pair managers + bobMgr = keeping.Manager(ks=bobKS, salt=bobSalt) + delMgr = keeping.Manager(ks=delKS, salt=delSalt) + watMgr = keeping.Manager(ks=watKS, salt=watSalt) + + # Init Keverys + bobKvy = eventing.Kevery(db=bobDB) + delKvy = eventing.Kevery(db=delDB) + watKvy = eventing.Kevery(db=watDB) + + # Setup Wat with own inception event + verfers, digers = watMgr.incept(stem='wat', temp=True) # algo default salty and rooted + + watSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + watPre = watSrdr.pre + watMgr.move(old=verfers[0].qb64, new=watPre) # move key pair label to prefix + # Setup wat's prefixes so wat's KEL will be Kever.locallyOwned() + watDB.prefixes.add(watPre) + assert watPre in watDB.prefixes + # setup wat's on kel + sigers = watMgr.sign(ser=watSrdr.raw, verfers=verfers) + msg = bytearray(watSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + watIcpMsg = msg # save for later + + # apply msg to wats's Kevery + psr.parse(ims=bytearray(watIcpMsg), kvy=watKvy, local=True) + watK = watKvy.kevers[watPre] + assert watK.prefixer.qb64 == watPre + assert watK.serder.said == watSrdr.said + + # Setup Bob with own inception event + verfers, digers = bobMgr.incept(stem='bob', temp=True) # algo default salty and rooted + bobSrdr = eventing.incept(keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers], + code=coring.MtrDex.Blake3_256) + + bobPre = bobSrdr.pre + bobMgr.move(old=verfers[0].qb64, new=bobPre) # move key pair label to prefix + # Setup Bob's prefixes so bob's KEL will be Kever.locallyOwned() and + # Del's KEL will be Kever.locallyDelegated() + bobDB.prefixes.add(bobPre) + assert bobPre in bobDB.prefixes + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=verfers) + msg = bytearray(bobSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + bobIcpMsg = msg # save for later + + # apply msg to bob's Kevery + psr.parse(ims=bytearray(bobIcpMsg), kvy=bobKvy, local=True) + bobK = bobKvy.kevers[bobPre] + assert bobK.prefixer.qb64 == bobPre + assert bobK.serder.said == bobSrdr.said + assert bobK.sn == 0 + + # apply msg to del's Kevery so he knows about the AID + psr.parse(ims=bytearray(bobIcpMsg), kvy=delKvy, local=True) + assert bobK.prefixer.qb64 in delKvy.kevers + delBobK = bobKvy.kevers[bobPre] # bobs kever in dels kevery + assert delBobK.sn == 0 + + # Setup Del's inception event assuming that Bob's next event will be + # an ixn delegating event + verfers, digers = delMgr.incept(stem='del', temp=True) # algo default salty and rooted + delSrdr = eventing.delcept(keys=[verfer.qb64 for verfer in verfers], + delpre=bobPre, + ndigs=[diger.qb64 for diger in digers]) + + delPre = delSrdr.pre + delMgr.move(old=verfers[0].qb64, new=delPre) # move key pair label to prefix + # Setup Del's prefixes so Del's KEL will be Kever.locallyOwned() + delDB.prefixes.add(delPre) + assert delPre in delDB.prefixes + + # Now create delegating event for Bob + seal = eventing.SealEvent(i=delPre, + s=delSrdr.ked["s"], + d=delSrdr.said) + bobSrdr = eventing.interact(pre=bobK.prefixer.qb64, + dig=bobK.serder.said, + sn=bobK.sn+1, + data=[seal._asdict()]) + + sigers = bobMgr.sign(ser=bobSrdr.raw, verfers=bobK.verfers) + + msg = bytearray(bobSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + bobIxnMsg1 = msg # delegating event with attachments + + # apply msg to bob's Kevery + psr.parse(ims=bytearray(bobIxnMsg1), kvy=bobKvy, local=True) + assert bobK.serder.said == bobSrdr.said # key state updated so event was validated + assert bobK.sn == 1 + + # now create Del's delegated inception event msg + sigers = delMgr.sign(ser=delSrdr.raw, verfers=verfers) + + msg = bytearray(delSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + counter = core.Counter(core.Codens.SealSourceCouples, + count=1, gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + seqner = coring.Seqner(sn=bobK.sn) + msg.extend(seqner.qb64b) + msg.extend(bobSrdr.saidb) + delIcpMsg = msg + + # apply Del's delegated inception event message to bob's Kevery + # because the attachment includes valid source seal then the Delegables + # escrow is bypassed and is validated and shows up in AES + psr.parse(ims=bytearray(delIcpMsg), kvy=bobKvy, local=True) + + # bobKvy.process(ims=bytearray(msg)) # process local copy of msg + assert delPre in bobKvy.kevers # successfully validated + bobDelK = bobKvy.kevers[delPre] # delK in bobs kevery + assert bobDelK.delegated + assert bobDelK.serder.said == delSrdr.said # key state updated so event was validated + couple = bobKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + assert couple == seqner.qb64b + bobSrdr.saidb + + # apply Del's inception msg to Del's Kevery + # Because locallyOwned by delegate event does not validate delegation + # and ignores the attached source seal + psr.parse(ims=bytearray(delIcpMsg), kvy=delKvy, local=True) + + assert delPre in delKvy.kevers + delK = delKvy.kevers[delPre] + # no AES entry for del's own delegated event when locallyOwned + assert not delKvy.db.getAes(dbing.dgKey(delPre, delSrdr.said)) + + # apply Del's delegated inception event message to wats's Kevery as remote + # because the attachment includes valid source seal but wat does not + # yet have Bob's delegating event entry. The event goes into partial + # delegated event escrow + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + # TEST MISSING EVENT SOURCE + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + esr = watKvy.db.esrs.get(keys=dgkey) + assert esr is not None + watKvy.db.esrs.rem(keys=dgkey) + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + dtb = watKvy.db.getDts(dgkey) + assert dtb is not None + watKvy.db.delDts(dgkey) # remove date timestamp of event so it will be invalid + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + eraw = watKvy.db.getEvt(dgkey) + assert eraw is not None + watKvy.db.delEvt(dgkey) # remove event data so it will be invalid + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + for (epre,), esn, edig in watKvy.db.pdes.getOnItemIter(): + dgkey = dgKey(epre, edig) + sigs = watKvy.db.getSigs(dgkey) + assert sigs is not None + watKvy.db.delSigs(dgkey) # remove event sigs so it will be invalid + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(delIcpMsg), kvy=watKvy, local=False) + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + watKvy.processEscrowPartialDels() + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 1 + assert escrows[0] == delSrdr.said # escrow entry for event + + watKvy.TimeoutPWE = 0 + time.sleep(0.001) + watKvy.processEscrowPartialDels() + assert not bobPre in watKvy.kevers + assert not delPre in watKvy.kevers + escrows = watKvy.db.pdes.getOn(keys=delPre, on=delSrdr.sn) + assert len(escrows) == 0 + + +def test_unverified_witness_escrow_validation_errors(): + """ + Test unverified witness escrow validation errors + + cam is controller + van is validator + wes is a witness + wok is a witness + wam is a witness + + """ + salt = core.Salter(raw=b'abcdef0123456789').qb64 + + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ + habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ + habbing.openHby(name="wes", base="test", salt=salt) as wesHby, \ + habbing.openHby(name="wak", base="test", salt=salt) as wokHby, \ + habbing.openHby(name="wam", base="test", salt=salt) as wamHby, \ + habbing.openHby(name="wil", base="test", salt=salt) as wilHby: + + # witnesses first so can set up inception event for cam + wsith = '1' + # setup Wes's habitat nontrans + # Wes's receipts will be rcts with a receipt couple attached + + wesHab = wesHby.makeHab(name='wes', isith=wsith, icount=1, transferable=False) + assert not wesHab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + wesKvy = eventing.Kevery(db=wesHab.db, lax=False, local=False) + + # setup Wok's habitat nontrans + # Wok's receipts will be rcts with a receipt couple attached + wokHab = wokHby.makeHab(name='wok', isith=wsith, icount=1, transferable=False) + assert not wokHab.kever.prefixer.transferable + # create non-local kevery for Wok to process nonlocal msgs + wokKvy = eventing.Kevery(db=wokHab.db, lax=False, local=False) + + # setup Wam's habitat nontrans + # Wams's receipts will be rcts with a receipt couple attached + wamHab = wamHby.makeHab(name='wam', isith=wsith, icount=1, transferable=False) + assert not wamHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wamKvy = eventing.Kevery(db=wamHab.db, lax=False, local=False) + + # setup Wil's habitat nontrans + # Wil's receipts will be rcts with a receipt couple attached + wilHab = wilHby.makeHab(name='wil', isith=wsith, icount=1, transferable=False) + assert not wilHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wilKvy = eventing.Kevery(db=wilHab.db, lax=False, local=False) + + # setup Cam's habitat trans multisig + wits = [wesHab.pre, wokHab.pre, wamHab.pre] + csith = '2' # hex str of threshold int + camHab = camHby.makeHab(name='cam', isith=csith, icount=3, toad=2, wits=wits,) + assert camHab.kever.prefixer.transferable + assert len(camHab.iserder.berfers) == len(wits) + for werfer in camHab.iserder.berfers: + assert werfer.qb64 in wits + assert camHab.kever.wits == wits + assert camHab.kever.toader.num == 2 + assert camHab.kever.sn == 0 + + # create non-local kevery for Cam to process onlocal msgs + camKvy = eventing.Kevery(db=camHab.db, lax=False, local=False) + + # setup Van's habitat trans multisig + vsith = '2' # two of three signing threshold + vanHab = vanHby.makeHab(name='van', isith=vsith, icount=3) + assert vanHab.kever.prefixer.transferable + # create non-local kevery for Van to process nonlocal msgs + vanKvy = eventing.Kevery(db=vanHab.db, lax=False, local=False) + + # make list so easier to batch + camWitKvys = [wesKvy, wokKvy, wamKvy] + camWitHabs = [wesHab, wokHab, wamHab] + + # Create Cam inception and send to each of Cam's witnesses + camIcpMsg = camHab.makeOwnInception() + rctMsgs = [] # list of receipts from each witness + for i in range(len(camWitKvys)): + kvy = camWitKvys[i] + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) + assert kvy.kevers[camHab.pre].sn == 0 # accepted event + assert len(kvy.cues) >= 1 # at least queued receipt cue + # better to find receipt cue in cues exactly + hab = camWitHabs[i] + rctMsg = hab.processCues(kvy.cues) # process cue returns rct msg + assert len(rctMsg) == 626 + rctMsgs.append(rctMsg) + + for msg in rctMsgs: # process rct msgs from all witnesses + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) + for hab in camWitHabs: + assert hab.pre in camKvy.kevers + + # get from Cam database copies of witness receipts received by Cam + # and send to witnesses so all witnesses have full set of receipts + # from all other witnesses + # reply one event or receipt one event with all witness attachments + dgkey = dbing.dgKey(pre=camHab.pre, dig=camHab.kever.serder.said) + wigs = camHab.db.getWigs(dgkey) + assert len(wigs) == 3 + wigers = [indexing.Siger(qb64b=bytes(wig)) for wig in wigs] + rserder = eventing.receipt(pre=camHab.pre, + sn=camHab.kever.sn, + said=camHab.kever.serder.said) + camIcpWitRctMsg = eventing.messagize(serder=rserder, wigers=wigers) + assert len(camIcpWitRctMsg) == 413 + for i in range(len(camWitKvys)): + kvy = camWitKvys[i] + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=kvy, local=True) + assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed + assert len(kvy.cues) == 0 # no cues + + # send Cam icp and witness rcts to Van + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy, local=True) + # should escrow since not witnesses + assert camHab.pre not in vanKvy.kevers + + # TEST MISSING DATE TIME + # process receipts + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy, local=True) + + assert camHab.pre not in vanKvy.kevers + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 3 + + for ekey, ecouple in vanKvy.db.getUweItemIter(): + pre, sn = splitSnKey(ekey) + rdiger, _wiger = deWitnessCouple(ecouple) + dgkey = dgKey(pre, bytes(rdiger.qb64b)) + vanKvy.db.delDts(dgkey) + vanKvy.processEscrowUnverWitness() + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 0 + + # TEST STALE EVENT + # process receipts + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy, local=True) + assert camHab.pre not in vanKvy.kevers + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 3 + + vanKvy.TimeoutUWE = 0 + time.sleep(0.001) + vanKvy.processEscrowUnverWitness() + escrows = vanKvy.db.getUwes(dbing.snKey(camHab.pre, 0)) # should be three receipts, one per witness + assert len(escrows) == 0 + + vanKvy.TimeoutUWE = 3600 + + +def test_unverified_nontransferable_receiptors_escrow_validation_errors(): + """ + Test unverified witness escrow validation errors + + cam is controller + van is validator + wes is a witness + wok is a witness + wam is a witness + + """ + salt = core.Salter(raw=b'abcdef0123456789').qb64 + + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ + habbing.openHby(name="badEvt", base="test", salt=salt) as badEvtHby, \ + habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ + habbing.openHby(name="wes", base="test", salt=salt) as wesHby, \ + habbing.openHby(name="wak", base="test", salt=salt) as wokHby, \ + habbing.openHby(name="wam", base="test", salt=salt) as wamHby, \ + habbing.openHby(name="wil", base="test", salt=salt) as wilHby: + + # witnesses first so can set up inception event for cam + wsith = '1' + # setup Wes's habitat nontrans + # Wes's receipts will be rcts with a receipt couple attached + + wesHab = wesHby.makeHab(name='wes', isith=wsith, icount=1, transferable=False) + assert not wesHab.kever.prefixer.transferable + # create non-local kevery for Wes to process nonlocal msgs + wesKvy = eventing.Kevery(db=wesHab.db, lax=False, local=False) + + # setup Wok's habitat nontrans + # Wok's receipts will be rcts with a receipt couple attached + wokHab = wokHby.makeHab(name='wok', isith=wsith, icount=1, transferable=False) + assert not wokHab.kever.prefixer.transferable + # create non-local kevery for Wok to process nonlocal msgs + wokKvy = eventing.Kevery(db=wokHab.db, lax=False, local=False) + + # setup Wam's habitat nontrans + # Wams's receipts will be rcts with a receipt couple attached + wamHab = wamHby.makeHab(name='wam', isith=wsith, icount=1, transferable=False) + assert not wamHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wamKvy = eventing.Kevery(db=wamHab.db, lax=False, local=False) + + # setup Wil's habitat nontrans + # Wil's receipts will be rcts with a receipt couple attached + wilHab = wilHby.makeHab(name='wil', isith=wsith, icount=1, transferable=False) + assert not wilHab.kever.prefixer.transferable + # create non-local kevery for Wam to process nonlocal msgs + wilKvy = eventing.Kevery(db=wilHab.db, lax=False, local=False) + + # setup Cam's habitat trans multisig + wits = [wesHab.pre, wokHab.pre, wamHab.pre] + csith = '2' # hex str of threshold int + camHab = camHby.makeHab(name='cam', isith=csith, icount=3, toad=2, wits=wits,) + assert camHab.kever.prefixer.transferable + assert len(camHab.iserder.berfers) == len(wits) + for werfer in camHab.iserder.berfers: + assert werfer.qb64 in wits + assert camHab.kever.wits == wits + assert camHab.kever.toader.num == 2 + assert camHab.kever.sn == 0 + + # create non-local kevery for Cam to process onlocal msgs + camKvy = eventing.Kevery(db=camHab.db, lax=False, local=False) + + # setup Van's habitat trans multisig + vsith = '2' # two of three signing threshold + vanHab = vanHby.makeHab(name='van', isith=vsith, icount=3) + assert vanHab.kever.prefixer.transferable + # create non-local kevery for Van to process nonlocal msgs + vanKvy = eventing.Kevery(db=vanHab.db, lax=False, local=False) + + # make list so easier to batch + camWitKvys = [wesKvy, wokKvy, wamKvy] + camWitHabs = [wesHab, wokHab, wamHab] + + # Create Cam inception and send to each of Cam's witnesses + camIcpMsg = camHab.makeOwnInception() + rctMsgs = [] # list of receipts from each witness + for i in range(len(camWitKvys)): + kvy = camWitKvys[i] + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) + assert kvy.kevers[camHab.pre].sn == 0 # accepted event + assert len(kvy.cues) >= 1 # at least queued receipt cue + # better to find receipt cue in cues exactly + hab = camWitHabs[i] + rctMsg = hab.processCues(kvy.cues) # process cue returns rct msg + assert len(rctMsg) == 626 + rctMsgs.append(rctMsg) + + for msg in rctMsgs: # process rct msgs from all witnesses + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) + + for hab in camWitHabs: + assert hab.pre in camKvy.kevers + + # send receipts one at a time to Van to escrow. Van not yet recieved + # icp event from Cam so not accepted Cam's pre + # compute keys for latest event in Cam's key state + dgkey = dbing.dgKey(pre=camHab.pre, dig=camHab.kever.serder.said) + snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) + + # TEST MISSING DATE TIME + # Van process rct msgs from all witnesses for Cam's icp message + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + + vanKvy.processEscrowUnverNonTrans() + + for ekey, etriplet in vanKvy.db.getUreItemIter(key=b''): + pre, sn = splitSnKey(ekey) + rsaider, _sprefixer, _cigar = deReceiptTriple(etriplet) + dgkey = dgKey(pre, bytes(rsaider.qb64b)) + dtb = vanKvy.db.getDts(dgkey) + if dtb is not None: + vanKvy.db.delDts(dgkey) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + + vanKvy.processEscrows() # process escrows + assert vanKvy.db.cntPwes(snkey) == 0 # nothing in partial witness escrow + assert vanKvy.db.cntUres(snkey) == 0 # still in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # still not accepted + + # TEST STALE EVENT + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + + vanKvy.TimeoutURE = 0 + time.sleep(0.001) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + + vanKvy.TimeoutURE = 3600 # reset timeout to normal value + + # TEST INVALID EVENT + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + vanKvy.processEscrowUnverNonTrans() + + # put value in db to trigger other validation errors + to_clean_up = [] + for ekey, etriplet in vanKvy.db.getUreItemIter(key=b''): + pre, sn = splitSnKey(ekey) + rsaider, _sprefixer, _cigar = deReceiptTriple(etriplet) + vanKvy.db.addKe(snKey(pre, sn), rsaider.qb64b) # add arbitrary value as key event + to_clean_up.append(snKey(pre, sn)) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + + # cleanup + for ureSnKey in to_clean_up: + vanKvy.db.delKes(ureSnKey) # remove value to clean up for later tests + + # TEST BAD ESCROWED RECEIPT + # put different inception event in the db so the escrow thinks the receipt is bad + # Set up bad event hab + badEvtHab = badEvtHby.makeHab(name='badEvt', isith='1', icount=1, toad=0, wits=[]) + badEvtIcpMsg = badEvtHab.makeOwnInception() + + for i, msg in enumerate(rctMsgs): + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) + # escrows to Ure + assert vanKvy.db.cntUres(snkey) == i + 1 # escrows + assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow + assert vanKvy.db.cntWigs(dgkey) == 0 # no wigs yet + assert camHab.pre not in vanKvy.kevers # not accepted + for hab in camWitHabs: # Van accepted icp events for Cam's witnesses + assert hab.pre in vanKvy.kevers + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 3 # three receipts in escrow + + # put value in db to trigger other validation errors + to_clean_up_ke = [] + to_clean_up_evt = [] + evt = serdering.SerderKERI(raw=camIcpMsg) + for ekey, etriplet in vanKvy.db.getUreItemIter(key=b''): + pre, sn = splitSnKey(ekey) + rsaider, _sprefixer, _cigar = deReceiptTriple(etriplet) + dig = rsaider.qb64b + dgkey = dgKey(pre, dig) # events use dgkey + vanKvy.db.addKe(snKey(pre, sn), rsaider.qb64b) # add mock key event digest + vanKvy.db.putEvt(dgkey, badEvtIcpMsg) # add mock key event + + to_clean_up_ke.append(snKey(pre, sn)) + to_clean_up_evt.append(dgkey) + vanKvy.processEscrowUnverNonTrans() + escrows = vanKvy.db.getUres(dbing.snKey(camHab.pre, 0)) + assert len(escrows) == 0 + for ureSnKey in to_clean_up_ke: + vanKvy.db.delKes(ureSnKey) # remove value to clean up for later tests + for ureEvtKey in to_clean_up_evt: + vanKvy.db.delEvt(ureEvtKey) + + +def test_delegables_escrow_validation_errors(): + gateSalt = core.Salter(raw=b'0123456789abcdef').qb64 + torSalt = core.Salter(raw=b'0123456789defabc').raw + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + + with habbing.openHby(name="delegate", temp=True, salt=gateSalt) as gateHby, \ + habbing.openHab(name="delegator", temp=True, salt=torSalt) as (torHby, torHab): + + gateHab = gateHby.makeHab(name="repTest", transferable=True, delpre=torHab.pre) + assert gateHab.pre == "EFqw1EgGdd2B6MgNLJaNO13_JoQpxAtasIjySDzGm9pd" + + gateIcp = gateHab.makeOwnEvent(sn=0) + torKvy = eventing.Kevery(db=torHab.db, lax=False, local=False) + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + # Exercise the MissingDelegableApprovalError case + torKvy.processEscrowDelegables() + + # TEST MISSING EVENT SOURCE + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + torKvy.db.esrs.rem(keys=dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + dtb = torKvy.db.getDts(dgkey) + assert dtb is not None + torKvy.db.delDts(dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + # TEST STALE EVENT + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + torKvy.TimeoutOOE = 0 + time.sleep(0.001) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + torKvy.TimeoutOOE = 3600 + + # TEST MISSING EVENT DATA + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + eraw = torKvy.db.getEvt(dgkey) + assert eraw is not None + torKvy.db.delEvt(dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + parsing.Parser().parse(ims=bytearray(gateIcp), kvy=torKvy, local=True) # tell delegate about delegator + assert gateHab.pre not in torKvy.kevers # should not be in delegate KELs until delegation finishes (seal arrives) + snkey = snKey(gateHab.kever.serder.preb, gateHab.kever.serder.sn) + assert len(torHab.db.delegables.get(keys=snkey)) == 1 + + for (pre, sn), dig in torKvy.db.delegables.getItemIter(): + edig = dig.encode() + dgkey = dgKey(pre.encode(), edig) + sigers = torKvy.db.getSigs(dgkey) + assert sigers is not None + torKvy.db.delSigs(dgkey) + torKvy.processEscrowDelegables() + escrows = torKvy.db.delegables.get(snkey) + assert len(escrows) == 0 + + +def test_query_not_found_escrow_validation_errors(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + with habbing.openHby() as inqHby, \ + habbing.openHby() as subjHby: + inqHab = inqHby.makeHab(name="inquisitor") + subHab = subjHby.makeHab(name="subject") + psr = parsing.Parser() + icp = inqHab.makeOwnInception() + subKvy = eventing.Kevery(db=subHab.db, lax=False, local=False) + assert inqHab.pre not in subHab.kevers + + inqKvy = eventing.Kevery(db=inqHab.db, lax=False, local=True) + assert subHab.pre not in inqHab.kevers + + qry = inqHab.query(subHab.pre, route="ksn", src=inqHab.pre) + + # TEST MISSING DATE TIME + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + for (pre, said), edig in inqKvy.db.qnfs.getItemIter(): + dgkey = dgKey(pre.encode(), edig.encode()) + dtb = inqKvy.db.getDts(dgkey) + assert dtb is not None + inqKvy.db.delDts(dgkey) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + inqKvy.TimeoutQNF = 0 + time.sleep(0.001) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + inqKvy.TimeoutQNF = 3600 + + # TEST MISSING EVENT DATA + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + for (pre, said), edig in inqKvy.db.qnfs.getItemIter(): + dgkey = dgKey(pre.encode(), edig.encode()) + eraw = inqKvy.db.getEvt(dgkey) + assert eraw is not None + inqKvy.db.delEvt(dgkey) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + for (pre, said), edig in inqKvy.db.qnfs.getItemIter(): + dgkey = dgKey(pre.encode(), edig.encode()) + sigs = inqKvy.db.getSigs(dgkey) + assert sigs is not None + inqKvy.db.delSigs(dgkey) + inqKvy.processQueryNotFound() + escrows = inqKvy.db.qnfs.get(dgkey) + assert len(escrows) == 0 + + # Exercise QueryNotFoundError + psr.parseOne(ims=bytearray(qry), kvy=inqKvy) # reapply query to inquisitor's escrow + inqKvy.processQueryNotFound() + + +def test_unverified_transferable_receiptors_escrow_validation_errors(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + + # init event DB and keep DB + with basing.openDB(name="ned") as db, keeping.openKS(name="ned") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Set up identifier for event creator, 2 of 3 weighted threshold multisig AID + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='edy', temp=True) + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # Create CESR message for inception event with local key signatures + msg = bytearray(srdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = msg + + # create transferable receipter (validator) inception keys 2 of 3 + rverfers, rdigers = mgr.incept(icount=3, ncount=3, stem='rob', temp=True) + rsith = '2' + + # create receipter's inception event + rsrdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=rsith, nsith=rsith, + keys=[verfer.qb64 for verfer in rverfers], + ndigs=[diger.qb64 for diger in rdigers]) + rpre = rsrdr.ked["i"] + ricpdig = rsrdr.said + mgr.move(old=rverfers[0].qb64, new=rpre) # move receipter key pair label to prefix + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + # CESR message for receipter inception event with local key signatures + msg = bytearray(rsrdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in rsigers: + msg.extend(siger.qb64b) + ricpmsg = msg + + # create transferable receipt of inception message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=0, said=icpdig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rcticpmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # Process receipt by kvy + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + kvy.processEscrowUnverTrans() + + # TEST MISSING DATE TIME + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.getDts(dgkey) + kvy.db.delDts(dgkey) + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + + kvy.TimeoutVRE = 0 + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + kvy.TimeoutVRE = 3600 + + # TEST MISSING RECEIPTED EVENT + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) # reapply receipt to escrow + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + kvy.processEscrowUnverTrans() + + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.delDts(dgkey) # invalidate events to clear the escrow + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) # reapply receipt to escrow + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply inception event to escrow so key event findable + assert pre in kvy.kevers # icp event present for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.delEvt(dgkey) + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + +def test_unverified_tranferable_receiptors_escrow_invalid_event_data(): + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + + # init event DB and keep DB + with basing.openDB(name="ned") as db, keeping.openKS(name="ned") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + + # Set up identifier for event creator, 2 of 3 weighted threshold multisig AID + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='edy', temp=True) + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # Create CESR message for inception event with local key signatures + msg = bytearray(srdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = msg + + # create transferable receipter (validator) inception keys 2 of 3 + rverfers, rdigers = mgr.incept(icount=3, ncount=3, stem='rob', temp=True) + rsith = '2' + + # create receipter's inception event + rsrdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=rsith, nsith=rsith, + keys=[verfer.qb64 for verfer in rverfers], + ndigs=[diger.qb64 for diger in rdigers]) + rpre = rsrdr.ked["i"] + ricpdig = rsrdr.said + mgr.move(old=rverfers[0].qb64, new=rpre) # move receipter key pair label to prefix + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + # CESR message for receipter inception event with local key signatures + msg = bytearray(rsrdr.raw) # message body + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) # message attachments + for siger in rsigers: + msg.extend(siger.qb64b) + ricpmsg = msg + + # create transferable receipt of inception message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=0, said=icpdig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rcticpmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # Process receipt by kvy + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + kvy.processEscrowUnverTrans() + + # TEST INVALID EVENT DATA + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) # reapply receipt to escrow + psr.parse(ims=bytearray(icpmsg), kvy=kvy) # apply inception event to escrow so key event findable + assert pre in kvy.kevers # icp event present for pre (receipted) + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + kvy.processEscrowUnverTrans() + + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 3 + + for ekey, equinlet in kvy.db.getVreItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + esaider, sprefixer, sseqner, ssaider, siger = deTransReceiptQuintuple(equinlet) + dgkey = dgKey(pre, bytes(esaider.qb64b)) + kvy.db.setEvt(dgkey, ricpmsg) + kvy.processEscrowUnverTrans() + escrows = kvy.db.getVres(dbing.snKey(pre, 0)) # so escrowed receipts + assert len(escrows) == 0 + + +def test_duplicitous_escrow_validation_errors(): + """ + Tests the duplicitous event escrow with a duplicitous interaction event to test validation errors + properly remove duplicitous events from the duplicitous event escrow. + """ + logger.setLevel(logging.TRACE) # gives test coverage to trace level logging blocks + salt = core.Salter(raw=b'0123456789abcdef').qb64 # init Salter + psr = parsing.Parser() + + with basing.openDB(name="edy") as db, keeping.openKS(name="edy") as ks: + mgr = keeping.Manager(ks=ks, salt=salt) + kvy = eventing.Kevery(db=db) + # Set up identifier for event creator, 2 of 3 weighted threshold multisig AID + sith = ["1/2", "1/2", "1/2"] + nxtsith = ["1/2", "1/2", "1/2"] + verfers, digers = mgr.incept(icount=3, ncount=3, stem='edy', temp=True) + srdr = eventing.incept(code=coring.MtrDex.Blake3_256, isith=sith, nsith=nxtsith, + keys=[verfer.qb64 for verfer in verfers], + ndigs=[diger.qb64 for diger in digers]) + pre = srdr.ked["i"] + icpdig = srdr.said + mgr.move(old=verfers[0].qb64, new=pre) # move key pair label to prefix + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + # make CESR message for inception event to give to parser + msg = bytearray(srdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + icpmsg = msg + + # create transferable receipter (validator) inception keys 2 of 3 + rverfers, rdigers = mgr.incept(icount=3, ncount=3, stem='ray', temp=True) + rsith = '2' + + # create receipter's inception event + rsrdr = eventing.incept(keys=[verfer.qb64 for verfer in rverfers], + isith=rsith, + nsith=rsith, + ndigs=[diger.qb64 for diger in rdigers], + code=coring.MtrDex.Blake3_256) + + rpre = rsrdr.ked["i"] + ricpdig = rsrdr.said + mgr.move(old=rverfers[0].qb64, new=rpre) # move receipter key pair label to prefix + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + # make CESR message for inception event to give to parser + msg = bytearray(rsrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in rsigers: + msg.extend(siger.qb64b) + ricpmsg = msg + + + # create transferable receipt of inception message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=0, said=icpdig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rcticpmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # create interaction event + srdr = eventing.interact(pre=pre, dig=icpdig, sn=1, data=[]) + ixndig = srdr.said + sigers = mgr.sign(ser=srdr.raw, verfers=verfers) + + msg = bytearray(srdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in sigers: + msg.extend(siger.qb64b) + + ixnmsg = msg + + # create duplicitous interaction event + dummyAnchor = dict(i=pre, s=15, d=pre) + dupSrdr = eventing.interact(pre=pre, dig=icpdig, sn=1, data=[dummyAnchor]) + dupIxndig = dupSrdr.said + dupSigers = mgr.sign(ser=dupSrdr.raw, verfers=verfers) + + msg = bytearray(dupSrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(sigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in dupSigers: + msg.extend(siger.qb64b) + + dupIxnmsg = msg + + # Create rotation event of receipter + # get current keys as verfers and next digests as digers + rverfers, rdigers = mgr.rotate(pre=rpre, ncount=3, temp=True) + + rsrdr = eventing.rotate(pre=rpre, + keys=[verfer.qb64 for verfer in rverfers], + isith=rsith, + dig=ricpdig, + nsith=rsith, + ndigs=[diger.qb64 for diger in rdigers], + sn=1, + data=[]) + + rrotdig = rsrdr.said + + rsigers = mgr.sign(ser=rsrdr.raw, verfers=rverfers) + + msg = bytearray(rsrdr.raw) + counter = core.Counter(core.Codens.ControllerIdxSigs, + count=len(rsigers), gvrsn=kering.Vrsn_1_0) + msg.extend(counter.qb64b) + for siger in rsigers: + msg.extend(siger.qb64b) + + rrotmsg = msg + + # create receipt(s) of interaction message with receipter rotation message + # create chit receipt(s) of interaction message + seal = eventing.SealEvent(i=rpre, + s=rsrdr.ked["s"], + d=rsrdr.said) + reserder = eventing.receipt(pre=pre, sn=1, said=ixndig) + # sign event not receipt + resigers = mgr.sign(ser=srdr.raw, verfers=rverfers) + rctixnmsg = eventing.messagize(serder=reserder, sigers=resigers, seal=seal) + + # Process receipt by kvy + psr.parse(ims=bytearray(rcticpmsg), kvy=kvy) + psr.parse(ims=bytearray(rctixnmsg), kvy=kvy) + assert pre not in kvy.kevers # no events yet for pre + assert rpre not in kvy.kevers # no events yet for rpre (receipter) + # check escrows are back + assert len(kvy.db.getVres(dbing.snKey(pre, 0))) == 3 + assert len(kvy.db.getVres(dbing.snKey(pre, 1))) == 3 + + # apply inception msg to Kevery to process + psr.parse(ims=bytearray(icpmsg), kvy=kvy) + # kvy.process(ims=bytearray(icpmsg)) # process local copy of msg + assert pre in kvy.kevers # event accepted + kvr = kvy.kevers[pre] + assert kvr.serder.said == icpdig # key state updated so event was validated + assert kvr.sn == 0 # key state successfully updated + + # apply ixn msg to Kevery to process + psr.parse(ims=bytearray(ixnmsg), kvy=kvy) + # kvy.process(ims=bytearray(ixnmsg)) # process local copy of msg + assert kvr.serder.said == ixndig # key state updated so event was validated + assert kvr.sn == 1 # key state successfully updated + + # TEST MISSING EVENT SOURCE + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + esr = kvy.db.esrs.get(keys=dgkey) + assert esr is not None + kvy.db.esrs.rem(keys=dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING DATE TIME + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + dtb = kvy.db.getDts(dgkey) + assert dtb is not None + kvy.db.delDts(dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST STALE EVENT + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + kvy.TimeoutLDE = 0 + time.sleep(0.001) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + kvy.TimeoutLDE = 3600 # reset timeout to default + + # TEST MISSING EVENT DATA + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + eraw = kvy.db.getEvt(dgkey) + assert eraw is not None + kvy.db.delEvt(dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + # TEST MISSING SIGNATURES + psr.parse(ims=bytearray(dupIxnmsg), kvy=kvy) # apply duplicitous ixn msg to Kevery to process + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 1 + + for ekey, edig in kvy.db.getLdeItemIter(): + pre, sn = splitSnKey(ekey) # get pre and sn from escrow item + dgkey = dgKey(pre, bytes(edig)) + sigs = kvy.db.getSigs(dgkey) + assert sigs is not None + kvy.db.delSigs(dgkey) + kvy.processEscrowDuplicitous() + escrows = kvy.db.getLdes(dbing.snKey(pre, 1)) + assert len(escrows) == 0 + + if __name__ == "__main__": - test_unverified_receipt_escrow() + #test_unverified_receipt_escrow() + test_missing_delegator_escrow() diff --git a/tests/core/test_eventing.py b/tests/core/test_eventing.py index c6532e63f..2b799a984 100644 --- a/tests/core/test_eventing.py +++ b/tests/core/test_eventing.py @@ -10,17 +10,20 @@ import pytest from keri import kering +from keri.kering import Vrsn_1_0 from keri.app import habbing, keeping from keri.app.keeping import openKS, Manager +from keri import core +from keri.core import Signer, Counter, Codens from keri.core import coring, eventing, parsing, serdering -from keri.core.coring import (Diger, MtrDex, Matter, IdrDex, Indexer, - CtrDex, Counter, Salter, Siger, Cigar, - Seqner, Verfer, Signer, Prefixer, - generateSigners, IdxSigDex, DigDex) +from keri.core.coring import (Diger, MtrDex, Matter, + Cigar, + Seqner, Verfer, Prefixer, DigDex) +from keri.core.indexing import (IdrDex, IdxSigDex, Indexer, Siger) from keri.core.eventing import Kever, Kevery from keri.core.eventing import (SealDigest, SealRoot, SealBacker, SealEvent, SealLast, StateEvent, StateEstEvent) -from keri.core.eventing import (TraitDex, LastEstLoc, Serials, versify, +from keri.core.eventing import (TraitDex, LastEstLoc, Kinds, versify, simple, ample) from keri.core.eventing import (deWitnessCouple, deReceiptCouple, deSourceCouple, deReceiptTriple, @@ -1520,7 +1523,7 @@ def test_state(mockHelpingNowUTC): # use same salter for all but different path # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # State NonDelegated (key state notification) # create transferable key pair for controller of KEL @@ -1801,7 +1804,7 @@ def test_messagize(): """ Test messagize utility function """ - salter = Salter(raw=b'0123456789abcdef') + salter = core.Salter(raw=b'0123456789abcdef') with openDB(name="edy") as db, openKS(name="edy") as ks: # Init key pair manager mgr = Manager(ks=ks, salt=salter.qb64) @@ -2062,7 +2065,7 @@ def test_kever(mockHelpingNowUTC): with openDB() as db: # Transferable case # Setup inception key event dict raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create current key sith = 1 # one signer # original signing keypair transferable default @@ -2090,7 +2093,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-digestive prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519}) + saids = {'i': coring.PreDex.Ed25519}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid @@ -2144,7 +2148,7 @@ def test_kever(mockHelpingNowUTC): # test exposeds raw = b"raw salt to test" # create signers with verfers - signers = coring.Salter(raw=raw).signers(count=3, path="next", temp=True) + signers = core.Salter(raw=raw).signers(count=3, path="next", temp=True) # create something to sign ser = b'abcdefghijklmnopqrstuvwxyz0123456789' @@ -2233,7 +2237,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-transferable prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519N}) + saids = {'i': coring.PreDex.Ed25519N}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid @@ -2244,7 +2249,7 @@ def test_kever(mockHelpingNowUTC): sad['n'] = nxt sad['bt'] = "{:x}".format(toad) - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EFsuiA86Q5gGuVOO3tou8KSU6LORSExIUxzWNrlnW7WP' assert serder.pre == skp0.verfer.qb64 aid0 = serder.pre @@ -2270,7 +2275,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-transferable prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519N}) + saids = {'i': coring.PreDex.Ed25519N}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid @@ -2278,7 +2284,7 @@ def test_kever(mockHelpingNowUTC): sad['kt'] = "{:x}".format(sith) # hex string sad['k'] = keys sad['nt'] = 0 - sad['n'] = nxt + sad['n'] = nxt # empty nxt sad['bt'] = "{:x}".format(toad) serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) @@ -2327,7 +2333,8 @@ def test_kever(mockHelpingNowUTC): # make with defaults with non-transferable prefix serder = serdering.SerderKERI(makify=True, ilk=kering.Ilks.icp, - saids = {'i': coring.PreDex.Ed25519N}) + saids = {'i': coring.PreDex.Ed25519N}, + verify=False) sad = serder.sad sad['i'] = skp0.verfer.qb64 # non-digestive aid @@ -2339,7 +2346,7 @@ def test_kever(mockHelpingNowUTC): sad['bt'] = "{:x}".format(toad) sad['b'] = baks - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EKcREpfNupJ8oOqdnqDIyJVr1-GgIMBrVOtBUR9Gm6lO' assert serder.pre == skp0.verfer.qb64 @@ -2362,7 +2369,7 @@ def test_kever(mockHelpingNowUTC): sad =serder.sad # makes copy sad['bt'] = "{:x}".format(toad) - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EBKhptvqccp0KNBaS45bNPdTE4m19U1IvweHJW2PIEDI' assert serder.pre == skp0.verfer.qb64 @@ -2391,7 +2398,7 @@ def test_kever(mockHelpingNowUTC): sad['b'] = baks sad['a'] = a - serder = serdering.SerderKERI(makify=True, verify=True, sad=sad) + serder = serdering.SerderKERI(makify=True, verify=False, sad=sad) assert serder.said == 'EEu-cdj_9b_66XRJ5UuhgEvJxAPpn4RjyaHvRgDU3iyA' assert serder.pre == skp0.verfer.qb64 @@ -2447,7 +2454,7 @@ def test_keyeventsequence_0(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2717,7 +2724,7 @@ def test_keyeventsequence_1(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2814,7 +2821,7 @@ def test_multisig_digprefix(): # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) pubkeys = [signer.verfer.qb64 for signer in signers] assert pubkeys == ['DErocgXD2RGSyvn3MObcx59jeOsEQhv2TqHirVkzrp0Q', @@ -2845,7 +2852,7 @@ def test_multisig_digprefix(): # create sig counter count = len(keys) - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i) for i in range(count)] # create key event verifier state @@ -2881,7 +2888,7 @@ def test_multisig_digprefix(): sn=1) # create sig counter count = len(keys) - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - count) for i in range(count, count + count)] # update key event verifier state @@ -2897,7 +2904,7 @@ def test_multisig_digprefix(): dig=kever.serder.said, sn=2) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - count) for i in range(count, count + count)] # update key event verifier state @@ -2913,7 +2920,7 @@ def test_multisig_digprefix(): dig=kever.serder.said, sn=3) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - count) for i in range(count, count + count)] # update key event verifier state @@ -2933,7 +2940,7 @@ def test_multisig_digprefix(): dig=kever.serder.said, sn=4) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs, count=count) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, count=count, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization sigers = [signers[i].sign(serder.raw, index=i - 5) for i in range(5, 8)] # update key event verifier state @@ -2968,7 +2975,7 @@ def test_recovery(): """ # create signers salt = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - signers = generateSigners(salt=salt, count=8, transferable=True) + signers = core.Salter(raw=salt).signers(count=8) with openDB(name="controller") as conlgr, openDB(name="validator") as vallgr: event_digs = [] # list of event digs in sequence to verify against database @@ -2985,7 +2992,7 @@ def test_recovery(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # return siger # create key event verifier state @@ -3007,7 +3014,7 @@ def test_recovery(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # returns siger # update key event verifier state @@ -3026,7 +3033,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3048,7 +3055,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3067,7 +3074,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3086,7 +3093,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3105,7 +3112,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3128,7 +3135,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3147,7 +3154,7 @@ def test_recovery(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = signers[esn].sign(serder.raw, index=0) # update key event verifier state @@ -3199,7 +3206,7 @@ def test_receipt(): """ raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers coeSigners = salter.signers(count=8, path='coe', temp=True) @@ -3238,7 +3245,7 @@ def test_receipt(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) # return Siger if index @@ -3267,7 +3274,7 @@ def test_receipt(): valCigar = valSigner.sign(ser=serder.raw) # returns Cigar cause no index assert valCigar.qb64 == ('0BADE2aOlwLi6OCF-jzRWSPuaOo916ADjwhA92hBQ1km' 'LSSYdzDiZIpJNFf0uislNR8uhCbB6x2Y1I6rqbNeBXwF') - recnt = Counter(code=CtrDex.NonTransReceiptCouples, count=1) + recnt = Counter(code=Codens.NonTransReceiptCouples, count=1, gvrsn=Vrsn_1_0) assert recnt.qb64 == '-CAB' res.extend(reserder.raw) @@ -3296,7 +3303,7 @@ def test_receipt(): said=fake) # sign event not receipt valCigar = valSigner.sign(ser=serder.raw) # returns Cigar cause no index - recnt = Counter(code=CtrDex.NonTransReceiptCouples, count=1) + recnt = Counter(code=Codens.NonTransReceiptCouples, count=1, gvrsn=Vrsn_1_0) # attach to receipt msg stream res.extend(reserder.raw) res.extend(recnt.qb64b) @@ -3318,7 +3325,7 @@ def test_receipt(): said=fake) # sign event not receipt valCigar = valSigner.sign(ser=serder.raw) # returns Cigar cause no index - recnt = Counter(code=CtrDex.NonTransReceiptCouples, count=1) + recnt = Counter(code=Codens.NonTransReceiptCouples, count=1, gvrsn=Vrsn_1_0) # attach to receipt msg stream res.extend(reserder.raw) res.extend(recnt.qb64b) @@ -3348,7 +3355,7 @@ def test_receipt(): event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) # returns siger # extend key event stream @@ -3369,7 +3376,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3394,7 +3401,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3416,7 +3423,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3438,7 +3445,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3460,7 +3467,7 @@ def test_receipt(): sn=sn) event_digs.append(serder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[esn].sign(serder.raw, index=0) @@ -3496,7 +3503,7 @@ def test_direct_mode(): # but goes both ways once initiated. raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers coeSigners = salter.signers(count=8, path='coe', temp=True) @@ -3530,7 +3537,7 @@ def test_direct_mode(): coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # return Siger if index @@ -3563,7 +3570,7 @@ def test_direct_mode(): val_event_digs.append(valSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = valSigners[vesn].sign(valSerder.raw, index=0) # return Siger if index @@ -3744,7 +3751,7 @@ def test_direct_mode(): sn=csn) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # returns siger @@ -3838,7 +3845,7 @@ def test_direct_mode(): sn=csn) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) @@ -3962,7 +3969,7 @@ def test_direct_mode_cbor_mgpk(): # but goes both ways once initiated. raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers coeSigners = salter.signers(count=8, path='coe', temp=True) @@ -3989,14 +3996,14 @@ def test_direct_mode_cbor_mgpk(): coeSerder = incept(keys=[coeSigners[cesn].verfer.qb64], ndigs=[coring.Diger(ser=coeSigners[cesn + 1].verfer.qb64b).qb64], code=MtrDex.Blake3_256, - kind=Serials.cbor) + kind=Kinds.cbor) assert csn == int(coeSerder.ked["s"], 16) == 0 coepre = coeSerder.ked["i"] coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # return Siger if index @@ -4021,14 +4028,14 @@ def test_direct_mode_cbor_mgpk(): valSerder = incept(keys=[valSigners[vesn].verfer.qb64], ndigs=[coring.Diger(ser=valSigners[vesn + 1].verfer.qb64b).qb64], code=MtrDex.Blake3_256, - kind=Serials.mgpk) + kind=Kinds.mgpk) assert vsn == int(valSerder.ked["s"], 16) == 0 valpre = valSerder.ked["i"] val_event_digs.append(valSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = valSigners[vesn].sign(valSerder.raw, index=0) # return Siger if index @@ -4065,7 +4072,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=coeK.sn, said=coeK.serder.said, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign coe's event not receipt # look up event to sign from val's kever for coe coeIcpDig = bytes(valKevery.db.getKeLast(key=snKey(pre=coepre, sn=csn))) @@ -4116,7 +4123,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=10, said=fake, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign event not receipt siger = valSigners[vesn].sign(ser=coeIcpRaw, index=0) # return Siger if index @@ -4151,7 +4158,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=valK.prefixer.qb64, sn=valK.sn, said=valK.serder.said, - kind=Serials.cbor) + kind=Kinds.cbor) # sign vals's event not receipt # look up event to sign from coe's kever for val valIcpDig = bytes(coeKevery.db.getKeLast(key=snKey(pre=valpre, sn=vsn))) @@ -4204,10 +4211,10 @@ def test_direct_mode_cbor_mgpk(): dig=coeKever.serder.said, ndigs=[coring.Diger(ser=coeSigners[cesn + 1].verfer.qb64b).qb64], sn=csn, - kind=Serials.cbor) + kind=Kinds.cbor) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) # returns siger @@ -4246,7 +4253,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=coeK.sn, said=coeK.serder.said, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign coe's event not receipt # look up event to sign from val's kever for coe coeRotDig = bytes(valKevery.db.getKeLast(key=snKey(pre=coepre, sn=csn))) @@ -4298,10 +4305,10 @@ def test_direct_mode_cbor_mgpk(): coeSerder = interact(pre=coeKever.prefixer.qb64, dig=coeKever.serder.said, sn=csn, - kind=Serials.cbor) + kind=Kinds.cbor) coe_event_digs.append(coeSerder.said) # create sig counter - counter = Counter(CtrDex.ControllerIdxSigs) # default is count = 1 + counter = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # default is count = 1 # sign serialization siger = coeSigners[cesn].sign(coeSerder.raw, index=0) @@ -4338,7 +4345,7 @@ def test_direct_mode_cbor_mgpk(): reserder = receipt(pre=coeK.prefixer.qb64, sn=coeK.sn, said=coeK.serder.said, - kind=Serials.mgpk) + kind=Kinds.mgpk) # sign coe's event not receipt # look up event to sign from val's kever for coe coeIxnDig = bytes(valKevery.db.getKeLast(key=snKey(pre=coepre, sn=csn))) @@ -4437,7 +4444,7 @@ def test_process_nontransferable(): nsigs = 1 # one attached signature unspecified index #["v", "t", "d", "i", "s", "kt", "k", "nt", "n","bt", "b", "c", "a"] - ked0 = dict(v=versify(kind=Serials.json, size=0), + ked0 = dict(v=versify(kind=Kinds.json, size=0), t=Ilks.icp, d="", i=aid0.qb64, # qual base 64 prefix @@ -4453,8 +4460,6 @@ def test_process_nontransferable(): ) _, ked0 = coring.Saider.saidify(sad=ked0) - # verify derivation of aid0 from ked0 - assert aid0.verify(ked=ked0) # Serialize ked0 tser0 = serdering.SerderKERI(sad=ked0) @@ -4466,7 +4471,7 @@ def test_process_nontransferable(): assert skp0.verfer.verify(tsig0.raw, tser0.raw) # create attached sig counter - cnt0 = Counter(CtrDex.ControllerIdxSigs) + cnt0 = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # create packet msgb0 = bytearray(tser0.raw + cnt0.qb64b + tsig0.qb64b) @@ -4477,7 +4482,7 @@ def test_process_nontransferable(): del msgb0[:rser0.size] # strip off event from front # extract sig counter - rcnt0 = Counter(qb64=msgb0) + rcnt0 = Counter(qb64=msgb0, gvrsn=Vrsn_1_0) nrsigs = rcnt0.count assert nrsigs == 1 del msgb0[:len(rcnt0.qb64)] @@ -4493,9 +4498,6 @@ def test_process_nontransferable(): assert verfer.verify(rsig.raw, rser0.raw) del msgb0[:len(rsig.qb64)] - # verify pre - raid0 = Prefixer(qb64=rser0.pre) - assert raid0.verify(ked=rser0.ked) """ Done Test """ @@ -4525,7 +4527,7 @@ def test_process_transferable(): nsigs = 1 # one attached signature unspecified index - ked0 = dict(v=versify(kind=Serials.json, size=0), # version string + ked0 = dict(v=versify(kind=Kinds.json, size=0), # version string t=Ilks.icp, d="", # SAID i="", # qb64 prefix @@ -4541,8 +4543,9 @@ def test_process_transferable(): ) - # Use non digestive AID - aid0 = Prefixer(ked=ked0, code=MtrDex.Ed25519) + # Use non digestive AID made from keys[0] + aid0 = Prefixer(raw=Verfer(qb64=keys[0]).raw, code=MtrDex.Ed25519) + #aid0 = Prefixer(ked=ked0, code=MtrDex.Ed25519) assert aid0.code == MtrDex.Ed25519 assert aid0.qb64 == skp0.verfer.qb64 # update ked with pre @@ -4559,7 +4562,7 @@ def test_process_transferable(): assert skp0.verfer.verify(tsig0.raw, tser0.raw) # create attached sig counter - cnt0 = Counter(CtrDex.ControllerIdxSigs) + cnt0 = Counter(Codens.ControllerIdxSigs, gvrsn=Vrsn_1_0) # create packet msgb0 = bytearray(tser0.raw + cnt0.qb64b + tsig0.qb64b) @@ -4570,7 +4573,7 @@ def test_process_transferable(): del msgb0[:rser0.size] # strip off event from front # extract sig counter - rcnt0 = Counter(qb64=msgb0) + rcnt0 = Counter(qb64=msgb0, gvrsn=Vrsn_1_0) nrsigs = rcnt0.count assert nrsigs == 1 del msgb0[:len(rcnt0.qb64)] @@ -4586,10 +4589,6 @@ def test_process_transferable(): assert verfer.verify(rsig.raw, rser0.raw) del msgb0[:len(rsig.qb64)] - # verify pre - raid0 = Prefixer(qb64=rser0.pre) - assert raid0.verify(ked=rser0.ked) - # verify nxt digest from event is still valid digers=rser0.ndigers #assert rnxt1.includes(keys=nxtkeys) @@ -4652,7 +4651,7 @@ def test_process_manual(): index = 0 # create key event dict - ked0 = dict(v=versify(kind=Serials.json, size=0), + ked0 = dict(v=versify(kind=Kinds.json, size=0), t=Ilks.icp, d="", i=aidmat.qb64, # qual base 64 prefix @@ -4668,7 +4667,7 @@ def test_process_manual(): ) _, ked0 = coring.Saider.saidify(sad=ked0) - txsrdr = serdering.SerderKERI(sad=ked0, kind=Serials.json) + txsrdr = serdering.SerderKERI(sad=ked0, kind=Kinds.json) assert txsrdr.raw == (b'{"v":"KERI10JSON00012b_","t":"icp","d":"EKYHED-wvkYDZv4tNUF9qiC1kgnnGLS9YUU8' b'PCWig_n4","i":"DK-WsHD7MKfQpBjJ3B2GwjqY9z90G94uzMs7irCiT-dL","s":"0","kt":"1' b'","k":["DK-WsHD7MKfQpBjJ3B2GwjqY9z90G94uzMs7irCiT-dL"],"nt":"1","n":["EDcWJG' @@ -4726,7 +4725,7 @@ def test_reload_kever(mockHelpingNowUTC): Test reload Kever from keystate state message """ - with habbing.openHby(name="nat", base="test") as natHby: + with habbing.openHby(name="nat", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as natHby: # setup Nat's habitat using default salt multisig already incepts natHab = natHby.makeHab(name="nat", isith='2', icount=3) assert natHab.name == 'nat' @@ -4795,10 +4794,10 @@ def test_reload_kever(mockHelpingNowUTC): def test_load_event(mockHelpingNowUTC): - with habbing.openHby(name="tor", base="test") as torHby, \ - habbing.openHby(name="wil", base="test") as wilHby, \ - habbing.openHby(name="wan", base="test") as wanHby, \ - habbing.openHby(name="tee", base="test") as teeHby: + with habbing.openHby(name="tor", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as torHby, \ + habbing.openHby(name="wil", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as wilHby, \ + habbing.openHby(name="wan", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as wanHby, \ + habbing.openHby(name="tee", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as teeHby: wanKvy = Kevery(db=wanHby.db, lax=False, local=False) torKvy = Kevery(db=torHby.db, lax=False, local=False) @@ -4808,6 +4807,7 @@ def test_load_event(mockHelpingNowUTC): assert wanHab.pre == "BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP" msg = wanHab.makeOwnEvent(sn=0) parsing.Parser().parse(ims=msg, kvy=torKvy) + assert wanHab.pre in torKvy.kevers # Create Wil the witness, we'll use him later wilHab = wilHby.makeHab(name="wil", transferable=False) @@ -4816,14 +4816,16 @@ def test_load_event(mockHelpingNowUTC): torHab = torHby.makeHab(name="tor", icount=1, isith='1', ncount=1, nsith='1', wits=[wanHab.pre], toad=1) assert torHab.pre == "EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP" torIcp = torHab.makeOwnEvent(sn=0) + assert torHab.pre in torHab.kvy.kevers # Try to load event before Wan has seen it with pytest.raises(ValueError): _ = eventing.loadEvent(wanHab.db, torHab.pre, torHab.pre) - parsing.Parser().parse(ims=bytearray(torIcp), kvy=wanKvy) + # tor events are locallyWitnessed by wan so must process as local + parsing.Parser().parse(ims=bytearray(torIcp), kvy=wanHab.kvy, local=True) # process as local - wanHab.processCues(wanKvy.cues) # process cue returns rct msg + wanHab.processCues(wanHab.kvy.cues) # process cue returns rct msg evt = eventing.loadEvent(wanHab.db, torHab.pre, torHab.pre) assert evt == {'ked': {'a': [], 'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], @@ -4857,8 +4859,8 @@ def test_load_event(mockHelpingNowUTC): # Anchor Tee's inception event in Tor's KEL ixn = torHab.interact(data=[dict(i=teeHab.pre, s='0', d=teeHab.kever.serder.said)]) - parsing.Parser().parse(ims=bytearray(ixn), kvy=wanKvy) - wanHab.processCues(wanKvy.cues) # process cue returns rct msg + parsing.Parser().parse(ims=bytearray(ixn), kvy=wanHab.kvy, local=True) # give to wan must be local + wanHab.processCues(wanHab.kvy.cues) # process cue returns rct msg evt = eventing.loadEvent(wanHab.db, torHab.pre, torHab.kever.serder.said) assert evt == {'ked': {'a': [{'d': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', @@ -4882,8 +4884,8 @@ def test_load_event(mockHelpingNowUTC): 'witnesses': []} # Add seal source couple to Tee's inception before sending to Wan - counter = coring.Counter(code=coring.CtrDex.SealSourceCouples, - count=1) + counter = Counter(Codens.SealSourceCouples, + count=1, gvrsn=Vrsn_1_0) teeIcp.extend(counter.qb64b) seqner = coring.Seqner(sn=torHab.kever.sn) teeIcp.extend(seqner.qb64b) @@ -4894,14 +4896,51 @@ def test_load_event(mockHelpingNowUTC): nrct = wilHab.receipt(serder=teeHab.kever.serder) # Now Wan should be ready for Tee's inception - parsing.Parser().parse(ims=bytearray(teeIcp), kvy=wanKvy) - parsing.Parser().parse(ims=bytearray(rct), kvy=wanKvy) - parsing.Parser().parse(ims=bytearray(nrct), kvy=wanKvy) + parsing.Parser().parse(ims=bytearray(teeIcp), kvy=wanKvy, local=True) # local + parsing.Parser().parse(ims=bytearray(rct), kvy=wanHab.kvy, local=True) # local + parsing.Parser().parse(ims=bytearray(nrct), kvy=wanHab.kvy, local=True) # local + # ToDo XXXX fix it so cues are durable in db so can process cues from + # both and remote sources + wanHab.processCues(wanHab.kvy.cues) # process cue returns rct msg wanHab.processCues(wanKvy.cues) # process cue returns rct msg # Endorse Tee's inception event with Wan's Hab just so we have non-trans receipts evt = eventing.loadEvent(wanHab.db, teeHab.pre, teeHab.pre) + #assert evt == {'ked': {'a': [], + #'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], + #'bt': '1', + #'c': [], + #'d': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'di': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'i': 'EDnrWpxagMvr5BBCwCOh3q5M9lvurboZ66vxR-GnIgQo', + #'k': ['DLDlVl1H2Q138A5tftVRpyy834ejsY33BB71kXLRNP2h'], + #'kt': '1', + #'n': ['EBTtZqMkJOO4nf3cCt6SdezwkoCKtx2fGUKHeFApj_Yx'], + #'nt': '1', + #'s': '0', + #'t': 'dip', + #'v': 'KERI10JSON00018d_'}, + #'receipts': {'nontransferable': [{'prefix': 'BEXrSXVksXpnfno_Di6RBX2Lsr9VWRAihjLhowfjNOQQ', + #'signature': '0BCQOeNT3mwAHxh6mYU9K_B2VmbtjJh7_8115k4JrBPR3c4' + #'3jUSO197H2J73vWMi61qzOovNkSWQbnRx3NFnrk8I'}], + #'transferable': [{'prefix': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'said': 'EBOVJXs0trI76PRfvJB2fsZ56PrtyR6HrUT9LOBra8VP', + #'sequence': '0AAAAAAAAAAAAAAAAAAAAAAA', + #'signature': 'AADGbcmUNw_SX7OVNX-PQYl41UZx_pgJXHOoMWrcfmCDGgkc1-' + #'MqXJjMD9S9moJ-lpPL9-AiXgITemMZL_QYGzIA'}]}, + #'signatures': [{'index': 0, + #'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' + #'2NAi0DMuZeXnlvch6ZP0coO'}], + #'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', + #'sequence': 1}, + #'stored': True, + #'timestamp': '2021-01-01T00:00:00.000000+00:00', + #'witness_signatures': [{'index': 0, + #'signature': 'AABPMW3J1iZyMC-elPOkdIhddhZB_BJYHTdYv5SxcrOfJL_5igDVB6zKD' + #'AQiTj_cNa7oP-l6xSRRxwlHDwqgSwcB'}], + #'witnesses': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP']} + # no source seal in load assert evt == {'ked': {'a': [], 'b': ['BAbSj3jfaeJbpuqg0WtvHw31UoRZOnN_RZQYBwbAqteP'], 'bt': '1', @@ -4927,8 +4966,6 @@ def test_load_event(mockHelpingNowUTC): 'signatures': [{'index': 0, 'signature': 'AAC1-NTntZ0xkgHwooNcKxe9G4XC-rgkSryVz0B_QrZR2kkv4IKi7DMkfMBd4Eck-' '2NAi0DMuZeXnlvch6ZP0coO'}], - 'source_seal': {'said': 'EF7pHYN6XABC9znRdzprt5frW-MMry9rfrCI-_t5Y8VD', - 'sequence': 1}, 'stored': True, 'timestamp': '2021-01-01T00:00:00.000000+00:00', 'witness_signatures': [{'index': 0, @@ -4942,7 +4979,7 @@ def test_load_event(mockHelpingNowUTC): if __name__ == "__main__": # pytest.main(['-vv', 'test_eventing.py::test_keyeventfuncs']) #test_process_manual() - #test_keyeventsequence_0() + test_keyeventsequence_0() #test_process_transferable() #test_messagize() test_direct_mode() diff --git a/tests/core/test_indexing.py b/tests/core/test_indexing.py new file mode 100644 index 000000000..453dab432 --- /dev/null +++ b/tests/core/test_indexing.py @@ -0,0 +1,812 @@ +# -*- encoding: utf-8 -*- +""" +tests.core.test_indexing module + +""" +from dataclasses import dataclass, astuple, asdict +from collections import namedtuple +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 + +import pysodium + +import pytest + +from keri.kering import (EmptyMaterialError, RawMaterialError, ShortageError, + InvalidVarIndexError, ) + + +from keri.help import helping +from keri.help.helping import (sceil, intToB64, b64ToInt, + codeB64ToB2, codeB2ToB64, nabSextets) + +from keri.core import indexing +from keri.core.indexing import (IdrDex, IdxSigDex, IdxCrtSigDex, IdxBthSigDex, + Xizage, Indexer, Siger) + + +from keri.core.coring import (Verfer,) + +def test_indexer_class(): + """ + Test Indexer class + """ + + assert Indexer.Codes == { + 'Ed25519_Sig': 'A', + 'Ed25519_Crt_Sig': 'B', + 'ECDSA_256k1_Sig': 'C', + 'ECDSA_256k1_Crt_Sig': 'D', + 'ECDSA_256r1_Sig': 'E', + 'ECDSA_256r1_Crt_Sig': 'F', + 'Ed448_Sig': '0A', + 'Ed448_Crt_Sig': '0B', + 'Ed25519_Big_Sig': '2A', + 'Ed25519_Big_Crt_Sig': '2B', + 'ECDSA_256k1_Big_Sig': '2C', + 'ECDSA_256k1_Big_Crt_Sig': '2D', + 'ECDSA_256r1_Big_Sig': '2E', + 'ECDSA_256r1_Big_Crt_Sig': '2F', + 'Ed448_Big_Sig': '3A', + 'Ed448_Big_Crt_Sig': '3B', + 'TBD0': '0z', + 'TBD1': '1z', + 'TBD4': '4z', + } + + assert Indexer.Names == \ + { + 'A': 'Ed25519_Sig', + 'B': 'Ed25519_Crt_Sig', + 'C': 'ECDSA_256k1_Sig', + 'D': 'ECDSA_256k1_Crt_Sig', + 'E': 'ECDSA_256r1_Sig', + 'F': 'ECDSA_256r1_Crt_Sig', + '0A': 'Ed448_Sig', + '0B': 'Ed448_Crt_Sig', + '2A': 'Ed25519_Big_Sig', + '2B': 'Ed25519_Big_Crt_Sig', + '2C': 'ECDSA_256k1_Big_Sig', + '2D': 'ECDSA_256k1_Big_Crt_Sig', + '2E': 'ECDSA_256r1_Big_Sig', + '2F': 'ECDSA_256r1_Big_Crt_Sig', + '3A': 'Ed448_Big_Sig', + '3B': 'Ed448_Big_Crt_Sig', + '0z': 'TBD0', + '1z': 'TBD1', + '4z': 'TBD4' + } + + + assert IdrDex.Ed25519_Sig == 'A' + assert IdrDex.Ed25519_Crt_Sig == 'B' + assert IdrDex.ECDSA_256k1_Sig == 'C' + assert IdrDex.ECDSA_256k1_Crt_Sig == 'D' + assert IdrDex.ECDSA_256r1_Sig == 'E' + assert IdrDex.ECDSA_256r1_Crt_Sig == 'F' + assert IdrDex.Ed448_Sig == '0A' + assert IdrDex.Ed448_Crt_Sig == '0B' + assert IdrDex.Ed25519_Big_Sig == '2A' + assert IdrDex.Ed25519_Big_Crt_Sig == '2B' + assert IdrDex.ECDSA_256k1_Big_Sig == '2C' + assert IdrDex.ECDSA_256k1_Big_Crt_Sig == '2D' + assert IdrDex.ECDSA_256r1_Big_Sig == '2E' + assert IdrDex.ECDSA_256r1_Big_Crt_Sig == '2F' + assert IdrDex.Ed448_Big_Sig == '3A' + assert IdrDex.Ed448_Big_Crt_Sig == '3B' + assert IdrDex.TBD0 == '0z' + assert IdrDex.TBD1 == '1z' + assert IdrDex.TBD4 == '4z' + + assert asdict(IdxSigDex) == { + 'Ed25519_Sig': 'A', + 'Ed25519_Crt_Sig': 'B', + 'ECDSA_256k1_Sig': 'C', + 'ECDSA_256k1_Crt_Sig': 'D', + 'ECDSA_256r1_Sig': 'E', + 'ECDSA_256r1_Crt_Sig': 'F', + 'Ed448_Sig': '0A', + 'Ed448_Crt_Sig': '0B', + 'Ed25519_Big_Sig': '2A', + 'Ed25519_Big_Crt_Sig': '2B', + 'ECDSA_256k1_Big_Sig': '2C', + 'ECDSA_256k1_Big_Crt_Sig': '2D', + 'ECDSA_256r1_Big_Sig': '2E', + 'ECDSA_256r1_Big_Crt_Sig': '2F', + 'Ed448_Big_Sig': '3A', + 'Ed448_Big_Crt_Sig': '3B', + } + + assert IdxSigDex.Ed25519_Sig == 'A' + assert IdxSigDex.Ed25519_Crt_Sig == 'B' + assert IdxSigDex.ECDSA_256k1_Sig == 'C' + assert IdxSigDex.ECDSA_256k1_Crt_Sig == 'D' + assert IdxSigDex.ECDSA_256r1_Sig == 'E' + assert IdxSigDex.ECDSA_256r1_Crt_Sig == 'F' + assert IdxSigDex.Ed448_Sig == '0A' + assert IdxSigDex.Ed448_Crt_Sig == '0B' + assert IdxSigDex.Ed25519_Big_Sig == '2A' + assert IdxSigDex.Ed25519_Big_Crt_Sig == '2B' + assert IdxSigDex.ECDSA_256k1_Big_Sig == '2C' + assert IdxSigDex.ECDSA_256k1_Big_Crt_Sig == '2D' + assert IdxSigDex.ECDSA_256r1_Big_Sig == '2E' + assert IdxSigDex.ECDSA_256r1_Big_Crt_Sig == '2F' + assert IdxSigDex.Ed448_Big_Sig == '3A' + assert IdxSigDex.Ed448_Big_Crt_Sig == '3B' + + + assert asdict(IdxCrtSigDex) == { + 'Ed25519_Crt_Sig': 'B', + 'ECDSA_256k1_Crt_Sig': 'D', + 'ECDSA_256r1_Crt_Sig': 'F', + 'Ed448_Crt_Sig': '0B', + 'Ed25519_Big_Crt_Sig': '2B', + 'ECDSA_256k1_Big_Crt_Sig': '2D', + 'ECDSA_256r1_Big_Crt_Sig': '2F', + 'Ed448_Big_Crt_Sig': '3B', + } + + assert IdxCrtSigDex.Ed25519_Crt_Sig == 'B' + assert IdxCrtSigDex.ECDSA_256k1_Crt_Sig == 'D' + assert IdxCrtSigDex.ECDSA_256r1_Crt_Sig == 'F' + assert IdxCrtSigDex.Ed448_Crt_Sig == '0B' + assert IdxCrtSigDex.Ed25519_Big_Crt_Sig == '2B' + assert IdxCrtSigDex.ECDSA_256k1_Big_Crt_Sig == '2D' + assert IdxCrtSigDex.ECDSA_256r1_Big_Crt_Sig == '2F' + assert IdxCrtSigDex.Ed448_Big_Crt_Sig == '3B' + + + assert asdict(IdxBthSigDex) == { + 'Ed25519_Sig': 'A', + 'ECDSA_256k1_Sig': 'C', + 'ECDSA_256r1_Sig': 'E', + 'Ed448_Sig': '0A', + 'Ed25519_Big_Sig': '2A', + 'ECDSA_256k1_Big_Sig': '2C', + 'ECDSA_256r1_Big_Sig': '2E', + 'Ed448_Big_Sig': '3A', + } + + assert IdxBthSigDex.Ed25519_Sig == 'A' + assert IdxBthSigDex.ECDSA_256k1_Sig == 'C' + assert IdxBthSigDex.ECDSA_256r1_Sig == 'E' + assert IdxBthSigDex.Ed448_Sig == '0A' + assert IdxBthSigDex.Ed25519_Big_Sig == '2A' + assert IdxBthSigDex.ECDSA_256k1_Big_Sig == '2C' + assert IdxBthSigDex.ECDSA_256r1_Big_Sig == '2E' + assert IdxBthSigDex.Ed448_Big_Sig == '3A' + + + # first character of code with hard size of code + assert Indexer.Hards == { + 'A': 1, 'B': 1, 'C': 1, 'D': 1, 'E': 1, 'F': 1, 'G': 1, 'H': 1, 'I': 1, + 'J': 1, 'K': 1, 'L': 1, 'M': 1, 'N': 1, 'O': 1, 'P': 1, 'Q': 1, 'R': 1, + 'S': 1, 'T': 1, 'U': 1, 'V': 1, 'W': 1, 'X': 1, 'Y': 1, 'Z': 1, + 'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1, 'f': 1, 'g': 1, 'h': 1, 'i': 1, + 'j': 1, 'k': 1, 'l': 1, 'm': 1, 'n': 1, 'o': 1, 'p': 1, 'q': 1, 'r': 1, + 's': 1, 't': 1, 'u': 1, 'v': 1, 'w': 1, 'x': 1, 'y': 1, 'z': 1, + '0': 2, '1': 2, '2': 2, '3': 2, '4': 2, + } + + # Codes table with sizes of code (hard) and full primitive material + assert Indexer.Sizes == { + 'A': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'B': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'C': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'D': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'E': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + 'F': Xizage(hs=1, ss=1, os=0, fs=88, ls=0), + '0A': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), + '0B': Xizage(hs=2, ss=2, os=1, fs=156, ls=0), + '2A': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2B': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2C': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2D': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2E': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '2F': Xizage(hs=2, ss=4, os=2, fs=92, ls=0), + '3A': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), + '3B': Xizage(hs=2, ss=6, os=3, fs=160, ls=0), + '0z': Xizage(hs=2, ss=2, os=0, fs=None, ls=0), + '1z': Xizage(hs=2, ss=2, os=1, fs=76, ls=1), + '4z': Xizage(hs=2, ss=6, os=3, fs=80, ls=1), + } + + assert Indexer.Sizes['A'].hs == 1 # hard size + assert Indexer.Sizes['A'].ss == 1 # soft size + assert Indexer.Sizes['A'].os == 0 # other size + assert Indexer.Sizes['A'].fs == 88 # full size + assert Indexer.Sizes['A'].ls == 0 # lead size + + # verify first hs Sizes matches hs in Codes for same first char + for ckey in Indexer.Sizes.keys(): + assert Indexer.Hards[ckey[0]] == Indexer.Sizes[ckey].hs + + # verify all Codes have hs > 0 and ss > 0 and fs >= hs + ss if fs is not None + # verify os is part of ss + for val in Indexer.Sizes.values(): + assert val.hs > 0 and val.ss > 0 + assert val.os >= 0 and val.os < val.ss + if val.os: + assert val.os == val.ss // 2 + if val.fs is not None: + assert val.fs >= val.hs + val.ss + assert val.fs % 4 == 0 + + # Bizes maps bytes of sextet of decoded first character of code with hard size of code + # verify equivalents of items for Sizes and Bizes + for skey, sval in Indexer.Hards.items(): + ckey = codeB64ToB2(skey) + assert Indexer.Bards[ckey] == sval + + """End Test""" + + +def test_indexer(): + """ + Test Indexer instance + """ + + + with pytest.raises(EmptyMaterialError): + indexer = Indexer() + + # Test signatures + sig = (b"\x99\xd2<9$$0\x9fk\xfb\x18\xa0\x8c@r\x122.k\xb2\xc7\x1fp\x0e'm\x8f@" + b'\xaa\xa5\x8c\xc8n\x85\xc8!\xf6q\x91p\xa9\xec\xcf\x92\xaf)\xde\xca' + b'\xfc\x7f~\xd7o|\x17\x82\x1d\xd4$[\xf9\xef\xc0' + assert salter.stretch(tier=Tiers.low) == b'\xf8e\x80\xbaX\x08\xb9\xba\xc6\x1e\x84\r\x1d\xac\xa7\\\x82Wc@`\x13\xfd\x024t\x8ct\xd3\x01\x19\xe9' + assert salter.stretch(tier=Tiers.med) == b',\xf3\x8c\xbb\xe9)\nSQ\xec\xad\x8c9?\xaf\xb8\xb0\xb3\xcdB\xda\xd8\xb6\xf7\r\xf6D}Z\xb9Y\x16' + assert salter.stretch(tier=Tiers.high) == b'(\xcd\xc4\xb85\xcd\xe8:\xfc\x00\x8b\xfd\xa6\tj.y\x98\x0b\x04\x1c\xe3hBc!I\xe49K\x16-' + + """ Done Test """ + + +def test_gensignerswithsalter(): + """ + Test generating a set of signers using Salter + + """ + signers = Salter().signers(count=2, temp=True, transferable=False) + assert len(signers) == 2 + for signer in signers: + assert signer.verfer.code == MtrDex.Ed25519N + + # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) # raw salt + raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' + assert len(raw) == 16 + signers = Salter(raw=raw).signers(count=4) + assert len(signers) == 4 + for signer in signers: + assert signer.code == MtrDex.Ed25519_Seed + assert signer.verfer.code == MtrDex.Ed25519 + + sigkeys = [signer.qb64 for signer in signers] + assert sigkeys == ['AK8F6AAiYDpXlWdj2O5F5-6wNCCNJh2A4XOlqwR_HwwH', + 'AOs8-zNPPh0EhavdrCfCiTk9nGeO8e6VxUCzwdKXJAd0', + 'AHMBU5PsIJN2U9m7j0SGyvs8YD8fkym2noELzxIrzfdG', + 'AJZ7ZLd7unQ4IkMUwE69NXcvDO9rrmmRH_Xk3TPu9BpP'] + + """ End Test """ + +def test_cipher_closs(): + """ + Test class attributes of Cipher + """ + assert Cipher.Codex == CiXDex + + assert Cipher.Codes == \ + { + 'X25519_Cipher_L0': '4C', + 'X25519_Cipher_L1': '5C', + 'X25519_Cipher_L2': '6C', + 'X25519_Cipher_Big_L0': '7AAC', + 'X25519_Cipher_Big_L1': '8AAC', + 'X25519_Cipher_Big_L2': '9AAC', + 'X25519_Cipher_Seed': 'P', + 'X25519_Cipher_Salt': '1AAH', + 'X25519_Cipher_QB64_L0': '4D', + 'X25519_Cipher_QB64_L1': '5D', + 'X25519_Cipher_QB64_L2': '6D', + 'X25519_Cipher_QB64_Big_L0': '7AAD', + 'X25519_Cipher_QB64_Big_L1': '8AAD', + 'X25519_Cipher_QB64_Big_L2': '9AAD', + 'X25519_Cipher_QB2_L0': '4E', + 'X25519_Cipher_QB2_L1': '5E', + 'X25519_Cipher_QB2_L2': '6E', + 'X25519_Cipher_QB2_Big_L0': '7AAE', + 'X25519_Cipher_QB2_Big_L1': '8AAE', + 'X25519_Cipher_QB2_Big_L2': '9AAE' + } + + """End Test""" + + +def test_cipher(): + """ + Test Cipher subclass of Matter + """ + # conclusion never use box_seed_keypair always use sign_seed_keypair and + # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey + # is always the same. + + assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 + + # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' + b'\xde\x06\xc0+') + seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b + assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' + #b'AGDswxA8qdkb646JFZWUflm_OKUeF41iG2gTw3N4GwCs' + + # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' + saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b + assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' + + # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + with pytest.raises(kering.EmptyMaterialError): + cipher = Cipher() + + raw = pysodium.crypto_box_seal(seedqb64b, pubkey) # uses nonce so different everytime + cipher = Cipher(raw=raw) + assert cipher.code == MtrDex.X25519_Cipher_Seed + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == seedqb64b + + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Seed) + assert cipher.code == MtrDex.X25519_Cipher_Seed + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == seedqb64b + + # test .decrypt method needs qb64 + prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b + assert cipher.decrypt(prikey=prikeyqb64).qb64b == seedqb64b + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == seedqb64b + + cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b + assert cipher.decrypt(seed=cryptseedqb64).qb64b == seedqb64b + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == seedqb64b + + # wrong but shorter code so instance creation succeeds + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Salt) + assert cipher.code == MtrDex.X25519_Cipher_Salt + with pytest.raises(ValueError): # but decryption fails + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + + + raw = pysodium.crypto_box_seal(saltqb64b, pubkey) # uses nonce so different everytime + cipher = Cipher(raw=raw) + assert cipher.code == MtrDex.X25519_Cipher_Salt + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == saltqb64b + + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Salt) + assert cipher.code == MtrDex.X25519_Cipher_Salt + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == saltqb64b + + with pytest.raises(kering.RawMaterialError): # wrong code to big for raw bytes + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_Seed) + + # test .decrypt method needs qb64 + prikeyqb64 = Matter(raw=prikey, code=MtrDex.X25519_Private).qb64b + assert cipher.decrypt(prikey=prikeyqb64).qb64b == saltqb64b + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == saltqb64b + + cryptseedqb64 = Matter(raw=cryptseed, code=MtrDex.Ed25519_Seed).qb64b + assert cipher.decrypt(seed=cryptseedqb64).qb64b == saltqb64b + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == saltqb64b + + with pytest.raises(kering.InvalidCodeError): # bad code + cipher = Cipher(raw=raw, code=MtrDex.Ed25519N) + + # Test bad raw size + raw = pysodium.crypto_box_seal(seedqb64b, pubkey) # uses nonce so different everytime + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw[:len(raw)-1]) # make raw too small + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw + b'_') # make raw too big + + raw = pysodium.crypto_box_seal(saltqb64b, pubkey) # uses nonce so different everytime + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw[:len(raw)-1]) # make raw too small + with pytest.raises(kering.InvalidSizeError): + cipher = Cipher(raw=raw + b'_') # make raw too big + + # variable sized ciphers + # Always init with L0 code and let Matter auto correct the code for the + # actual lead size of raw + + # qb64 lead 0 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb64b + texter = core.Texter(qb64b=uncb) + assert texter.text == plain + + # qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dogcats" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L1 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb64b + texter = core.Texter(qb64b=uncb) + assert texter.text == plain + + # qb64 lead 2 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L2 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb64b + texter = core.Texter(qb64b=uncb) + assert texter.text == plain + + # qb2 lead 0 (always lead 0 when qb2 from texter) + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb2, pubkey) # uses nonce so different everytime + assert len(raw) == 96 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB2_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB2_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == texter.qb2 + texter = core.Texter(qb2=uncb) + assert texter.text == plain + + # sniffable qb64 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L1 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb64 lead 2 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L2 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == strm + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb2 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.GenericGroup + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime + assert len(raw) == 93 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == strmb2 + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb2=strmb2, strip=True) + assert texter.text == plain + + + # sniffable qb2 lead 0 Big + + plain = "The quick brown fox jumps over the lazy" * 324 + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime + assert len(raw) == 12696 + assert (3 - (len(raw) % 3)) % 3 == 0 + assert (len(raw) // 3 ) > (64 ** 2 - 1) # triplets + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_Big_L0 + uncb = pysodium.crypto_box_seal_open(cipher.raw, pubkey, prikey) + assert uncb == strmb2 + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + texter = core.Texter(qb2=strmb2, strip=True) + assert texter.text == plain + + # test .decrypt method with variable sized qb64 coded ciphers + # qb64 lead 0 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L0 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb64b + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb64b + + + # qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dogcats" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L1 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb64b + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb64b + + + # qb64 lead 2 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb64b, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L2 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb64b + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb64b + + # test .decrypt method with variable sized qb2 coded ciphers + # qb2 lead 0 (always lead 0 when qb2 from texter) + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + raw = pysodium.crypto_box_seal(texter.qb2, pubkey) # uses nonce so different everytime + assert len(raw) == 96 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_QB2_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB2_L0 + # test using prikey + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(prikey=prikeyqb64) + texter = cipher.decrypt(prikey=prikeyqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == texter.qb2 + # test using seed + with pytest.raises(kering.InvalidCodeError): # needs klas when var len qb64 + result = cipher.decrypt(seed=cryptseedqb64) + texter = cipher.decrypt(seed=cryptseedqb64, klas=core.Texter) + assert texter.text == plain + # test with bare + texter = cipher.decrypt(seed=cryptseedqb64, bare=True) == texter.qb2 + + # sniffable qb64 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b # sniffable stream + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 108 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strm + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strm + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strm + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strm + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + + # sniffable qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 116 + assert (3 - (len(raw) % 3)) % 3 == 1 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L1 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strm + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strm + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strm + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strm + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb64 lead 2 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + strm = counter.qb64b + texter.qb64b + raw = pysodium.crypto_box_seal(strm, pubkey) # uses nonce so different everytime + assert len(raw) == 112 + assert (3 - (len(raw) % 3)) % 3 == 2 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L2 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strm + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strm + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strm + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strm + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream + strm = bytearray(strm) + counter = core.Counter(qb64b=strm, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb64b=strm, strip=True) + assert texter.text == plain + + # sniffable qb2 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.GenericGroup + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime + assert len(raw) == 93 + assert (3 - (len(raw) % 3)) % 3 == 0 + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strmb2 + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strmb2 + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) + assert counter.code == core.CtrDex_2_0.GenericGroup + texter = core.Texter(qb2=strmb2, strip=True) + assert texter.text == plain + + # sniffable qb2 lead 0 Big + + plain = "The quick brown fox jumps over the lazy" * 324 + texter = core.Texter(text=plain) + assert texter.text == plain + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + strm = counter.qb64b + texter.qb64b + strmb2 = decodeB64(strm) + raw = pysodium.crypto_box_seal(strmb2, pubkey) # uses nonce so different everytime + assert len(raw) == 12696 + assert (3 - (len(raw) % 3)) % 3 == 0 + assert (len(raw) // 3 ) > (64 ** 2 - 1) # triplets + cipher = Cipher(raw=raw, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_Big_L0 + # test using prikey + streamer = cipher.decrypt(prikey=prikeyqb64) # default klas is streamer + assert streamer.stream == strmb2 + streamer = cipher.decrypt(prikey=prikeyqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(prikey=prikeyqb64, bare=True) == streamer.stream + # test using seed + streamer = cipher.decrypt(seed=cryptseedqb64) + assert streamer.stream == strmb2 + streamer = cipher.decrypt(seed=cryptseedqb64, klas=core.Streamer) + assert streamer.stream == strmb2 + # test with bare + assert cipher.decrypt(seed=cryptseedqb64, bare=True) == streamer.stream + strmb2 = bytearray(strmb2) + counter = core.Counter(qb2=strmb2, strip=True) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + texter = core.Texter(qb2=strmb2, strip=True) + assert texter.text == plain + + """ Done Test """ + + +def test_encrypter(): + """ + Test Encrypter subclass of Matter + """ + # conclusion never use box_seed_keypair always use sign_seed_keypair and + # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey + # is always the same. + + assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 + + # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' + b'\xde\x06\xc0+') + seedqb64b = Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b + assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' + + # salt = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + salt = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' + saltqb64b = Matter(raw=salt, code=MtrDex.Salt_128).qb64b + assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' + + # seed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + with pytest.raises(kering.EmptyMaterialError): + encrypter = Encrypter() + + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + assert encrypter.verifySeed(seed=cryptsigner.qb64) + + verfer = Verfer(raw=verkey, code=MtrDex.Ed25519) + + encrypter = Encrypter(verkey=verfer.qb64) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + encrypter = Encrypter(verkey=verfer.qb64b) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + # user Prefixer to generate original verkey + prefixer = Prefixer(qb64=verfer.qb64) + encrypter = Encrypter(verkey=prefixer.qb64b) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + # Test encrypt method + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + assert encrypter.verifySeed(seed=cryptsigner.qb64) + + cipher = encrypter.encrypt(ser=seedqb64b, code=MtrDex.X25519_Cipher_Seed) + assert cipher.code == MtrDex.X25519_Cipher_Seed + uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) + assert uncb == seedqb64b + + cipher = encrypter.encrypt(ser=saltqb64b, code=MtrDex.X25519_Cipher_Salt) + assert cipher.code == MtrDex.X25519_Cipher_Salt + uncb = pysodium.crypto_box_seal_open(cipher.raw, encrypter.raw, prikey) + assert uncb == saltqb64b + + # needs tests of encrypter with prim param instead of ser (see roundtrip) + + + """ Done Test """ + + +def test_decrypter(): + """ + Test Decrypter subclass of Matter + """ + # conclusion never use box_seed_keypair always use sign_seed_keypair and + # then use crypto_sign_xk_to_box_xk to generate x25519 keys so the prikey + # is always the same. + + assert pysodium.crypto_box_SEEDBYTES == pysodium.crypto_sign_SEEDBYTES == 32 + + # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) + seed = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' + b'\xde\x06\xc0+') + signer = Signer(raw=seed, code=MtrDex.Ed25519_Seed) + assert signer.verfer.code == MtrDex.Ed25519 + assert signer.verfer.transferable # default + seedqb64b = signer.qb64b + assert seedqb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' + # also works for Matter + assert seedqb64b == Matter(raw=seed, code=MtrDex.Ed25519_Seed).qb64b + + # raw = pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) + raw = b'6\x08d\r\xa1\xbb9\x8dp\x8d\xa0\xc0\x13J\x87r' + salter = Salter(raw=raw, code=MtrDex.Salt_128) + assert salter.code == MtrDex.Salt_128 + saltqb64b = salter.qb64b + assert saltqb64b == b'0AA2CGQNobs5jXCNoMATSody' + # also works for Matter + assert saltqb64b == Matter(raw=raw, code=MtrDex.Salt_128).qb64b # + + # cryptseed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + cryptsigner = Signer(raw=cryptseed, code=MtrDex.Ed25519_Seed, transferable=True) + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + with pytest.raises(kering.EmptyMaterialError): + decrypter = Decrypter() + + # create encrypter + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + # create cipher of seed + seedcipher = encrypter.encrypt(ser=seedqb64b, code=MtrDex.X25519_Cipher_Seed) + assert seedcipher.code == MtrDex.X25519_Cipher_Seed + # each encryption uses a nonce so not a stable representation for testing + + # create decrypter from prikey + decrypter = Decrypter(raw=prikey) + assert decrypter.code == MtrDex.X25519_Private + assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' + assert decrypter.raw == prikey + + # decrypt seed cipher using ser + designer = decrypter.decrypt(qb64=seedcipher.qb64b, transferable=signer.verfer.transferable) + assert designer.qb64b == seedqb64b + assert designer.code == MtrDex.Ed25519_Seed + assert designer.verfer.code == MtrDex.Ed25519 + assert signer.verfer.transferable + + # test bare decryption returns plain not instance + plain = decrypter.decrypt(qb64=seedcipher.qb64b, + transferable=signer.verfer.transferable, + bare=True) + assert plain == seedqb64b + + # decrypt seed cipher using cipher + designer = decrypter.decrypt(cipher=seedcipher, transferable=signer.verfer.transferable) + assert designer.qb64b == seedqb64b + assert designer.code == MtrDex.Ed25519_Seed + assert designer.verfer.code == MtrDex.Ed25519 + assert signer.verfer.transferable + + # create cipher of salt + saltcipher = encrypter.encrypt(ser=saltqb64b, code=MtrDex.X25519_Cipher_Salt) + assert saltcipher.code == MtrDex.X25519_Cipher_Salt + # each encryption uses a nonce so not a stable representation for testing + + # decrypt salt cipher using ser + desalter = decrypter.decrypt(qb64=saltcipher.qb64b) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + # test bare decryption returns plain not instance + plain = decrypter.decrypt(qb64=saltcipher.qb64b, bare=True) + assert plain == saltqb64b + + # decrypt salt cipher using cipher + desalter = decrypter.decrypt(cipher=saltcipher) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + # use previously stored fully qualified seed cipher with different nonce + # get from seedcipher above + cipherseed = ('PM9jOGWNYfjM_oLXJNaQ8UlFSAV5ACjsUY7J16xfzrlpc9Ve3A5WYrZ4o_' + 'NHtP5lhp78Usspl9fyFdnCdItNd5JyqZ6dt8SXOt6TOqOCs-gy0obrwFkPPqBvVkEw') + designer = decrypter.decrypt(qb64=cipherseed, transferable=signer.verfer.transferable) + assert designer.qb64b == seedqb64b + assert designer.code == MtrDex.Ed25519_Seed + assert designer.verfer.code == MtrDex.Ed25519 + + # use previously stored fully qualified salt cipher with different nonce + # get from saltcipher above + ciphersalt = ('1AAHjlR2QR9J5Et67Wy-ZaVdTryN6T6ohg44r73GLRPnHw-5S3ABFkhWy' + 'IwLOI6TXUB_5CT13S8JvknxLxBaF8ANPK9FSOPD8tYu') + desalter = decrypter.decrypt(qb64=ciphersalt) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + # Create new decrypter but use seed parameter to init prikey + decrypter = Decrypter(seed=cryptsigner.qb64b) + assert decrypter.code == MtrDex.X25519_Private + assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' + assert decrypter.raw == prikey + + # decrypt ciphersalt + desalter = decrypter.decrypt(qb64=saltcipher.qb64b) + assert desalter.qb64b == saltqb64b + assert desalter.code == MtrDex.Salt_128 + + """ Done Test """ + +def test_roundtrip(): + """Test round trip encrypt decrypt with variable sized ciphers""" + + # cryptseed = pysodium.randombytes(pysodium.crypto_box_SEEDBYTES) + cryptseed = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' + verkey, sigkey = pysodium.crypto_sign_seed_keypair(cryptseed) # raw + pubkey = pysodium.crypto_sign_pk_to_box_pk(verkey) + prikey = pysodium.crypto_sign_sk_to_box_sk(sigkey) + + # create decrypter from prikey + decrypter = Decrypter(raw=prikey) + assert decrypter.code == MtrDex.X25519_Private + assert decrypter.qb64 == 'OLCFxqMz1z1UUS0TEJnvZP_zXHcuYdQsSGBWdOZeY5VQ' + assert decrypter.raw == prikey + + # create encrypter from pubkey + encrypter = Encrypter(raw=pubkey) + assert encrypter.code == MtrDex.X25519 + assert encrypter.qb64 == 'CAF7Wr3XNq5hArcOuBJzaY6Nd23jgtUVI6KDfb3VngkR' + assert encrypter.raw == pubkey + + + # Test cipher qb64 lead 0 + plain = "The quick brown fox jumps over the lazy " + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L0 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + # Test cipher qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dogcats" + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L1 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + # Test cipher qb64 lead 2 + plain = "The quick brown fox jumps over the lazy dog" + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB64_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB64_L2 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + + # Test cipher qb2 (always L0 when qb2) + plain = "The quick brown fox jumps over the lazy dog" + tin = core.Texter(text=plain) # texter in + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=tin, code=CiXDex.X25519_Cipher_QB2_L0) + assert cipher.code == CiXDex.X25519_Cipher_QB2_L0 + # decrypt cipher using Decrypter + tout = decrypter.decrypt(cipher=cipher, klas=core.Texter) # texter out + assert tout.text == tin.text + + + # sniffable qb64 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb64b + texter.qb64b) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + + + # sniffable qb64 lead 1 + plain = "The quick brown fox jumps over the lazy dog" + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb64b + texter.qb64b) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L1 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + + + # sniffable qb64 lead 2 + plain = "The quick brown fox jumps over the lazy " + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb64b + texter.qb64b) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L2 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + + + # sniffable qb2 lead 0 + plain = "The quick brown fox jumps over the lazy" + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + # sniffable streamer in + sin = core.Streamer(stream=counter.qb2 + texter.qb2) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_L0 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + + # sniffable qb2 lead 0 Big + plain = "The quick brown fox jumps over the lazy" * 324 + texter = core.Texter(text=plain) + counter = core.Counter(core.Codens.GenericGroup, count=texter.size) + assert counter.code == core.CtrDex_2_0.BigGenericGroup + # sniffable streamer in + sin = core.Streamer(stream=counter.qb2 + texter.qb2) + # create cipher using Encrypter + cipher = encrypter.encrypt(prim=sin, code=CiXDex.X25519_Cipher_L0) + assert cipher.code == CiXDex.X25519_Cipher_Big_L0 + # decrypt cipher using Decrypter + # sniffable stream out + sout = decrypter.decrypt(cipher=cipher, klas=core.Streamer) + assert sin.stream == sout.stream + + """End Test""" + + + +if __name__ == "__main__": + test_signer() + test_salter() + test_gensignerswithsalter() + test_cipher_closs() + test_cipher() + test_encrypter() + test_decrypter() + test_roundtrip() + diff --git a/tests/core/test_streaming.py b/tests/core/test_streaming.py new file mode 100644 index 000000000..86eefc399 --- /dev/null +++ b/tests/core/test_streaming.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +""" +tests.core.test_streaming module + +""" +from binascii import unhexlify + +import pytest + + +from keri import kering + +from keri.help import helping + +from keri.core import (Matter,) +from keri.core.coring import dumps + + +from keri.core import streaming +from keri.core.streaming import (annot, denot, Streamer) + + +def test_streamer(): + """Test streamer instance""" + pass + + """End Test""" + + +def test_annot(): + """Test annot function Annotate""" + + # simple Inception + ims = (b'-FAtYKERICAAXicpEEx4oxGYbNrd6nZsdGu2KdN4MSDGD5IWS7hXjST7r8ewDG9X' + b'hvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAAMAAB-IALDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQMAAA-IAAMAAA-IAA-IAA-IAA') + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms + print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + + # complex inception + ims = (b'-FDCYKERICAAXicpEKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiEKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAAAMAAC-IAhDG9XhvcVryHj' + b'oIGcj5nK4sAE3oslQHWi4fBJre3NGwTQDK58m521o6nwgcluK8Mu2ULvScXM9kB1' + b'bSORrxNSS9cnDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MAAC-IAh' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_EMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUGEEbufBpvagqe9kijKISOoQPYFEOpy22CZJGJqQZp' + b'ZEyPMAAD-IAhBG9XhvcVryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQBK58m521' + b'o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnBMOmBoddcrRHShSajb4d60S6RK34' + b'gXZ2WYbr3AiPY1M0-IABXDND-IA8-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAAAEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAABEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMAAPEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms + print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + + + # interaction + ims = (b'-FB6YKERICAAXixnEHXLwMJsZLyG643VW8Do1cqqiMxD_E65Mc3Z1we6vTaREKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAABEKIuA20I5q6IrgAHrX-g' + b'kAt4Og17Ebu5CDBrRvh8RToi-IBU-RAuDG9XhvcVryHjoIGcj5nK4sAE3oslQHWi' + b'4fBJre3NGwTQMAACEB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_DK58' + b'm521o6nwgcluK8Mu2ULvScXM9kB1bSORrxNSS9cnMAAiEMrowWRk6u1imR32ZNHn' + b'TPUtc7uSAvrchIPN3I8S6vUG-QAMMABDEEbufBpvagqe9kijKISOoQPYFEOpy22C' + b'ZJGJqQZpZEyP-RAXDMOmBoddcrRHShSajb4d60S6RK34gXZ2WYbr3AiPY1M0MACA' + b'EB9O4V-zUteZJJFubu1h0xMtzt0wuGpLMVj1sKVsElA_') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms + print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + + + # Rotation + ims = (b'-FCGYKERICAAXrotEDHlTlOcSXZInbTE4iXzb1iFjZcxJZn3C3UXhckb3uQmEKIu' + b'A20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToiMAACEHXLwMJsZLyG643VW8Do' + b'1cqqiMxD_E65Mc3Z1we6vTaRMAAC-IAhDH7p14xo09rob5cEupmo8jSDi35ZOGt1' + b'k4t2nm1C1A68DIAdqJzLWEwQbhXEMOFjvFVZ7oMCJP4XXDP_ILaTEBAQDKhYdMBe' + b'P6FoH3ajGJTf_4fH229rm_lTZXfYkfwGTMERMAAC-IAhEBvDSpcj3y0y9W2-1GzY' + b'J85KEkDIPxu4y_TxAK49k7ciEEb97lh2oOd_yM3meBaRX5xSs8mIeBoPdhOTgVkd' + b'31jbECQTrhKHgrOXJS4kdvifvOqoJ7RjfJSsN3nshclYStgaMAAD-IALBG9XhvcV' + b'ryHjoIGcj5nK4sAE3oslQHWi4fBJre3NGwTQ-IALBH7p14xo09rob5cEupmo8jSD' + b'i35ZOGt1k4t2nm1C1A68-IAA-IAA') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms + print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + + + # Delegated Inception + ims = (b'-FDeYKERICAAXdipEKCFMk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCEKCF' + b'Mk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCMAAA4AADA1s2c1s2c1s2-IAh' + b'DIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uVDN7WiKyjLLBTK92xayCu' + b'ddZsBuwPmD2BKrl83h1xEUtiDOE5jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl' + b'5nfY4AADA1s2c1s2c1s2-IAhEKFoJ9Conb37zSn8zHLKP3YwHbeQiD1D9Qx0MagJ' + b'44DSEC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZEHgewy_ymPxtSFwu' + b'X2KaI_mPmoIUkxClviX3f-M38kCDMAAD-IAhBIR8GACw4z2GC5_XoReU4DMKbqi6' + b'-EdbgDZUAobRb8uVBN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiBOE5' + b'jmI9ktNSAddEke1rH2cGMDq4uYmyagDkAzHl5nfY-IAA-IBI-RAuDIR8GACw4z2G' + b'C5_XoReU4DMKbqi6-EdbgDZUAobRb8uVMAADEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSDN7WiKyjLLBTK92xayCuddZsBuwPmD2BKrl83h1xEUtiMAAE' + b'EC7sCVf_rYJ_khIj7UdlzrtemP31TuHTPUsGjvWni8GZ-QAYMAAVEHgewy_ymPxt' + b'SFwuX2KaI_mPmoIUkxClviX3f-M38kCDMD4SEKFoJ9Conb37zSn8zHLKP3YwHbeQ' + b'iD1D9Qx0MagJ44DSEKIuA20I5q6IrgAHrX-gkAt4Og17Ebu5CDBrRvh8RToi') + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms + print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + + + # Delegated Rotatation + ims = (b'-FBaYKERICAAXdrtEKfRY6YrpqUU0HyKWMGvNtzuZCaeMcIBrdKzHAqpmtTAEKCF' + b'Mk4nmn3t8jdC1pB_-Qmp7w8EROvdYaxgru7vHOjCMAABEKCFMk4nmn3t8jdC1pB_' + b'-Qmp7w8EROvdYaxgru7vHOjCMAAB-IALDJ0pLe3f2zGus0Va1dqWAnukWdZHGNWl' + b'K9NciJop9N4fMAAB-IALENX_LTL97uOSOkA1PEzam9vtmCLPprnbcpi71wXpmhFF' + b'MAAD-IALBIR8GACw4z2GC5_XoReU4DMKbqi6-EdbgDZUAobRb8uV-IALBJ0pLe3f' + b'2zGus0Va1dqWAnukWdZHGNWlK9NciJop9N4f-IAA-IAA') + + + bms = bytearray(ims) # make copy + print(f"incoming = \n{ims}\n") + ams = annot(bms) # annotated message dream + assert not bms + print(f"annotated = \n{ams}\n") + dms = denot(ams) + assert dms == ims + + # JWK https://connect2id.com/products/nimbus-jose-jwt/examples/jwk-generation#okp + # ED25519 EDDSA + djwk = { + "kty" : "OKP", + "crv" : "Ed25519", + "x" : "11qYAYKxCrfVS_7TyWQHOg7hcvPapiMlrwIaaPcHURo", + "d" : "nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A", + "use" : "sig", + "kid" : "FdFYFzERwC2uCBB46pZQi4GG85LujR8obt-KWRBICVQ" + } + + jwk = dumps(djwk, kering.Kinds.json) + assert jwk == (b'{"kty":"OKP","crv":"Ed25519","x":"11qYAYKxCrfVS_7TyWQHOg7hcvPapiMlrwIaaPcHUR' + b'o","d":"nWGxne_9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A","use":"sig","kid":"FdFY' + b'FzERwC2uCBB46pZQi4GG85LujR8obt-KWRBICVQ"}') + + assert len(jwk) == 193 + assert 193 - 44 == 149 + + # ECDSA Example + # https://pycose.readthedocs.io/en/latest/pycose/keys/ec2.html + + ckat = { + 'KTY': 'EC2', + 'CURVE': 'P_256', + 'ALG': 'ES256', + 'D': unhexlify(b'57c92077664146e876760c9520d054aa93c3afb04e306705db6090308507b4d3') + } + + + ck = (b'\xa6\x01\x02\x03&\x01!X\xba\xc5\xb1\x1c\xad\x8f\x99\xf9\xc7+\x05\xcfK\x9e&\xd2D\xdc\x18\x9ftR(%Z!\x9a\x86\xd6\xa0\x9e\xffX\x13\x8b\xf8-\xc1\xb6\xd5b\xbe\x0f\xa5J\xb7\x80J:d\xb6\xd7,\xcf\xedko\xb6\xed(\xbb\xfc\x11~#XW\xc9 wfAF\xe8vv\x0c\x95\xd0T\xaa\x93\xc3\xaf\xb0N0g\x05\xdb`\x900\x85\x07\xb4\xd3') + assert len(ck) == 105 + assert 105 - 36 == 69 + + + + + """End Test""" + + + +if __name__ == "__main__": + test_streamer() + test_annot() + + + diff --git a/tests/core/test_structing.py b/tests/core/test_structing.py new file mode 100644 index 000000000..47f2066dc --- /dev/null +++ b/tests/core/test_structing.py @@ -0,0 +1,681 @@ +# -*- coding: utf-8 -*- +""" +tests.core.test_structing module + +""" +from dataclasses import dataclass, astuple, asdict +from typing import NamedTuple +from collections import namedtuple +from collections.abc import Mapping + + +import pytest + + +from keri import kering + +from keri.help import helping + +from keri.core import (Matter, Diger, Prefixer, Number) + + +from keri.core import structing +from keri.core.structing import (SealDigest, SealRoot, SealBacker, SealEvent, + SealLast, SealTrans) +from keri.core.structing import (Castage, + Structor, EClanDom, ECastDom, + Sealer, SClanDom, SCastDom, ) + + +def test_structor_doms(): + """ + test doms in structure + """ + assert EClanDom == structing.EmptyClanDom() + assert ECastDom == structing.EmptyCastDom() + assert SClanDom == structing.SealClanDom() + assert SCastDom == structing.SealCastDom() + + assert asdict(EClanDom) == {} + assert asdict(ECastDom) == {} + + assert asdict(SClanDom) == \ + { + 'SealDigest': SealDigest, + 'SealRoot': SealRoot, + 'SealBacker': SealBacker, + 'SealLast': SealLast, + 'SealTrans': SealTrans, + 'SealEvent': SealEvent, + } + + assert asdict(SCastDom) == \ + { + 'SealDigest': SealDigest(d=Castage(kls=Diger, prm=None)), + 'SealRoot': SealRoot(rd=Castage(kls=Diger, prm=None)), + 'SealBacker': SealBacker(bi=Castage(Prefixer, prm=None), + d=Castage(kls=Diger, prm=None)), + 'SealLast': SealLast(i=Castage(kls=Prefixer, prm=None)), + 'SealTrans': SealTrans(s=Castage(kls=Number, prm='numh'), + d=Castage(kls=Diger, prm=None)), + 'SealEvent': SealEvent(i=Castage(kls=Prefixer, prm=None), + s=Castage(kls=Number, prm='numh'), + d=Castage(kls=Diger, prm=None)) + } + + + """End Test""" + + + +def test_structor_class(): + """ + test Structor class variables etc + """ + assert Structor.Clans == EClanDom + assert Structor.Casts == ECastDom + assert Structor.Names == {} + + """End Test""" + +def test_structor(): + """ + test Structor instance + """ + + with pytest.raises(kering.InvalidValueError): + structor = Structor() # test default + + + aid = 'BN5Lu0RqptmJC-iXEldMMrlEew7Q01te2fLgqlbqW9zR' + prefixer = Prefixer(qb64=aid) + num = 14 + number = Number(num=num) + snq = number.qb64 + snh = number.numh + dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' + diger = Diger(qb64=dig) + + # Test with single field namedtuple for data + + data = SealDigest(d=diger) + clan = SealDigest + cast = SealDigest(d=Castage(Diger)) + crew = SealDigest(d=dig) + name = SealDigest.__name__ + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealDigest._fields + klas = data.__class__ + assert klas == clan + + qb64 = diger.qb64 + qb2 = diger.qb2 + + # Test data + structor = Structor(data=data) + assert structor.data == data + assert structor.clan == clan + assert structor.name == name + assert structor.cast == cast + assert structor.crew == crew + assert structor.asdict == data._asdict() + assert structor.asdict == {'d': diger} + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test cast + structor = Structor(cast=cast, crew=crew) + #assert structor.data == data different instances so not == + assert structor.clan == clan + assert structor.name == name + assert structor.cast == cast + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64) + assert structor.clan == clan + assert structor.name == name + assert structor.cast == cast + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64.encode()) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64.encode(), strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb64.encode()) + structor = Structor(cast=cast, qb64=ba, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + + structor = Structor(cast=cast, qb2=qb2) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb2=qb2, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb2) + structor = Structor(cast=cast, qb2=ba, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + + # Test clan and cast + structor = Structor(clan=clan, cast=cast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64.encode()) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb2=qb2) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test clan with cast and crew as dicts + structor = Structor(clan=clan, cast=dcast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + structor = Structor(cast=cast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=dcast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # creates custom clan since both cast and crew are dicts + structor = Structor(cast=dcast, crew=dcrew) + assert structor.data.__class__.__name__ == "d" + assert structor.clan != clan + assert structor.name == "d" + assert structor.cast == cast # tuple compare is by field value not type + assert structor.cast.__class__.__name__ == "d" + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + + # Test with multiple field namedtuple for data + + + data = SealEvent(i=prefixer, s=number, d=diger) + clan = SealEvent + cast = SealEvent(i=Castage(Prefixer), + s=Castage(Number, 'numh'), + d=Castage(Diger)) + + #naive cast doesn't know about prm for Number + ncast = SealEvent(i=Castage(Prefixer), + s=Castage(Number), + d=Castage(Diger)) + crew = SealEvent(i=aid, s=snh, d=dig) + + # naive crew does't know about prm for Number + ncrew = SealEvent(i=aid, s=snq, d=dig) + name = SealEvent.__name__ + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealEvent._fields + klas = data.__class__ + assert klas == clan + + qb64 = prefixer.qb64 + number.qb64 + diger.qb64 # ''.join(crew) + qb2 = prefixer.qb2 + number.qb2 + diger.qb2 + + # Test data + structor = Structor(data=data) + assert structor.data == data + assert structor.clan == clan + assert structor.cast == ncast + assert structor.crew == ncrew + assert structor.name == name + assert structor.asdict == data._asdict() + assert structor.asdict == \ + { + 'i': prefixer, + 's': number, + 'd': diger, + } + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test cast + structor = Structor(cast=cast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64.encode()) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb64=qb64.encode(), strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb64.encode()) + structor = Structor(cast=cast, qb64=ba, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + + structor = Structor(cast=cast, qb2=qb2) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=cast, qb2=qb2, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + ba = bytearray(qb2) + structor = Structor(cast=cast, qb2=ba, strip=True) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + assert not ba # stripped so empty + + # Test clan and cast + structor = Structor(clan=clan, cast=cast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb64=qb64.encode()) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(clan=clan, cast=cast, qb2=qb2) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + + # Test clan with cast and crew as dicts + structor = Structor(clan=clan, cast=dcast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + structor = Structor(cast=cast, crew=dcrew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + structor = Structor(cast=dcast, crew=crew) + assert structor.clan == clan + assert structor.cast == cast + assert structor.crew == crew + assert structor.name == name + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # creates custom clan since both cast and crew are dicts + structor = Structor(cast=dcast, crew=dcrew) + assert structor.data.__class__.__name__ == "i_s_d" + assert structor.clan != clan + assert structor.name == "i_s_d" + assert structor.cast == cast # tuple compare is by field value not type + assert structor.cast.__class__.__name__ == "i_s_d" + assert structor.crew == crew + assert structor.qb64 == qb64 + assert structor.qb64b == qb64.encode() + assert structor.qb2 == qb2 + + # Test no clan and cast or crew as dict + with pytest.raises(kering.EmptyMaterialError): + structor = Structor(cast=dcast) # missing crew + + with pytest.raises(kering.InvalidValueError): + structor = Structor(crew=crew) # missing cast + + """End Test""" + + + +def test_sealer_class(): + """ + test sealer class variables etc + """ + assert Sealer.Clans == SClanDom + assert Sealer.Casts == SCastDom + assert Sealer.Names == \ + { + ('d',): 'SealDigest', + ('rd',): 'SealRoot', + ('bi', 'd'): 'SealBacker', + ('i',): 'SealLast', + ('s', 'd'): 'SealTrans', + ('i', 's', 'd'): 'SealEvent', + } + + """End Test""" + + +def test_sealer(): + """ + test sealer instance + """ + + with pytest.raises(kering.InvalidValueError): + sealer = Sealer() # test default + + + aid = 'BN5Lu0RqptmJC-iXEldMMrlEew7Q01te2fLgqlbqW9zR' + prefixer = Prefixer(qb64=aid) + num = 14 + number = Number(num=num) + snq = number.qb64 + snh = number.snh + dig = 'ELC5L3iBVD77d_MYbYGGCUQgqQBju1o4x1Ud-z2sL-ux' + diger = Diger(qb64=dig) + + # Test with single field namedtuple for data + + data = SealDigest(d=diger) + clan = SealDigest + cast = SealDigest(d=Castage(Diger)) + crew = SealDigest(d=dig) + name = SealDigest.__name__ + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealDigest._fields + klas = data.__class__ + assert klas == clan + + qb64 = diger.qb64 + qb2 = diger.qb2 + + # Test data + sealer = Sealer(data=data) + assert sealer.data == data + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.asdict == data._asdict() + assert sealer.asdict == {'d': diger} + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + sealer = Sealer(crew=crew) # uses known cast + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + sealer = Sealer(crew=dcrew) # uses known cast + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # uses known class + sealer = Sealer(cast=dcast, crew=dcrew) + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast # tuple compare is by field value not type + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # Test with multiple field namedtuple for data + + data = SealEvent(i=prefixer, s=number, d=diger) + clan = SealEvent + cast = SealEvent(i=Castage(Prefixer), + s=Castage(Number, "numh"), + d=Castage(Diger)) + # naive cast since data does not provide prm for number + ncast = SealEvent(i=Castage(Prefixer), + s=Castage(Number), + d=Castage(Diger)) + + crew = SealEvent(i=aid, s=snh, d=dig) + # naive crew since data does not provide prm for number + ncrew = SealEvent(i=aid, s=snq, d=dig) + + name = SealEvent.__name__ + + dcast = cast._asdict() + dcrew = crew._asdict() + + assert data._fields == SealEvent._fields + klas = data.__class__ + assert klas == clan + + qb64 = prefixer.qb64 + number.qb64 + diger.qb64 # ''.join(crew) + qb2 = prefixer.qb2 + number.qb2 + diger.qb2 + + # Test data + sealer = Sealer(data=data) + assert sealer.data == data + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == ncast + assert sealer.crew == ncrew + assert sealer.asdict == data._asdict() + assert sealer.asdict == {'i': prefixer, + 's': number, + 'd': diger} + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # Test no clan but with one or the other of cast and crew as dict or namedtuple + sealer = Sealer(crew=crew) # uses known cast + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + sealer = Sealer(crew=dcrew) # uses known cast + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + # uses known class + sealer = Sealer(cast=dcast, crew=dcrew) + assert sealer.clan == clan + assert sealer.name == name + assert sealer.cast == cast # tuple compare is by field value not type + assert sealer.crew == crew + assert sealer.qb64 == qb64 + assert sealer.qb64b == qb64.encode() + assert sealer.qb2 == qb2 + + + +if __name__ == "__main__": + test_structor_doms() + test_structor_class() + test_structor() + test_sealer_class() + test_sealer() + + + diff --git a/tests/core/test_weighted_threshold.py b/tests/core/test_weighted_threshold.py index aac8c1995..8342f49d3 100644 --- a/tests/core/test_weighted_threshold.py +++ b/tests/core/test_weighted_threshold.py @@ -8,10 +8,14 @@ import pytest from keri import help -from keri.db import dbing, basing -from keri.app import keeping + +from keri import core, kering from keri.core import coring, eventing, parsing +from keri.db import basing +from keri.app import keeping + + logger = help.ogler.getLogger() @@ -20,7 +24,7 @@ def test_weighted(): Test multisig with weighted threshold """ - wesSalt = coring.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter + wesSalt = core.Salter(raw=b'0123456789abcdef').qb64 # init wes Salter # init event DB and keep DB with basing.openDB(name="wes") as wesDB, keeping.openKS(name="wes") as wesKS: @@ -49,8 +53,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -85,8 +89,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=wesK.verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -122,8 +126,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -168,8 +172,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) @@ -214,8 +218,8 @@ def test_weighted(): sigers = wesMgr.sign(ser=wesSrdr.raw, verfers=verfers) msg = bytearray(wesSrdr.raw) - counter = coring.Counter(coring.CtrDex.ControllerIdxSigs, - count=len(sigers)) + counter = core.Counter(core.Codens.ControllerIdxSigs, count=len(sigers), + gvrsn=kering.Vrsn_1_0) msg.extend(counter.qb64b) for siger in sigers: msg.extend(siger.qb64b) diff --git a/tests/core/test_witness.py b/tests/core/test_witness.py index 0c34e46b2..5457cb078 100644 --- a/tests/core/test_witness.py +++ b/tests/core/test_witness.py @@ -6,8 +6,12 @@ import os from keri import help + +from keri import core +from keri.core import eventing, parsing, serdering, indexing + from keri.app import habbing -from keri.core import coring, eventing, parsing, serdering + from keri.db import dbing logger = help.ogler.getLogger() @@ -24,7 +28,7 @@ def test_indexed_witness_replay(): wam is a witness """ - salt = coring.Salter(raw=b'abcdef0123456789').qb64 + salt = core.Salter(raw=b'abcdef0123456789').qb64 with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ @@ -95,7 +99,7 @@ def test_indexed_witness_replay(): rctMsgs = [] # list of receipts from each witness for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 0 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -105,7 +109,7 @@ def test_indexed_witness_replay(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -116,7 +120,7 @@ def test_indexed_witness_replay(): dgkey = dbing.dgKey(pre=camHab.pre, dig=camHab.kever.serder.said) wigs = camHab.db.getWigs(dgkey) assert len(wigs) == 3 - wigers = [coring.Siger(qb64b=bytes(wig)) for wig in wigs] + wigers = [indexing.Siger(qb64b=bytes(wig)) for wig in wigs] rserder = eventing.receipt(pre=camHab.pre, sn=camHab.kever.sn, said=camHab.kever.serder.said) @@ -124,16 +128,17 @@ def test_indexed_witness_replay(): assert len(camIcpWitRctMsg) == 413 for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=kvy, local=True) assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed assert len(kvy.cues) == 0 # no cues # send Cam icp and witness rcts to Van - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy, local=True) # should escrow since not witnesses assert camHab.pre not in vanKvy.kevers # process receipts - parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIcpWitRctMsg), kvy=vanKvy, local=True) + vanKvy.processEscrowUnverWitness() vanKvy.processEscrows() assert camHab.pre in vanKvy.kevers # now accepted vcKvr = vanKvy.kevers[camHab.pre] @@ -145,7 +150,7 @@ def test_indexed_witness_replay(): rctMsgs = [] # list of receipts from each witness for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 1 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -155,7 +160,7 @@ def test_indexed_witness_replay(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -166,7 +171,7 @@ def test_indexed_witness_replay(): dgkey = dbing.dgKey(pre=camHab.pre, dig=camHab.kever.serder.said) wigs = camHab.db.getWigs(dgkey) assert len(wigs) == 3 - wigers = [coring.Siger(qb64b=bytes(wig)) for wig in wigs] + wigers = [indexing.Siger(qb64b=bytes(wig)) for wig in wigs] rserder = eventing.receipt(pre=camHab.pre, sn=camHab.kever.sn, said=camHab.kever.serder.said) @@ -174,15 +179,15 @@ def test_indexed_witness_replay(): assert len(camIxnWitRctMsg) == 413 for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=kvy, local=True) assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed assert len(kvy.cues) == 0 # no cues # send Cam ixn's witness rcts to Van first then send Cam ixn - parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIxnWitRctMsg), kvy=vanKvy, local=True) vanKvy.processEscrows() assert vcKvr.sn == 0 - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy, local=True) assert vcKvr.sn == 0 vanKvy.processEscrows() assert vcKvr.sn == 1 @@ -192,13 +197,13 @@ def test_indexed_witness_replay(): # Cam update itself with Wil receipts including Wils inception camReplayMsg = camHab.replay() assert len(camReplayMsg) == 2038 - parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy) + parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy, local=True) assert camHab.pre in wilKvy.kevers assert wilKvy.kevers[camHab.pre].sn == 1 # asscepted both events assert len(wilKvy.cues) == 2 wilRctMsg = wilHab.processCues(wilKvy.cues) # process cue returns rct msg assert len(wilKvy.cues) == 0 - parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy, local=True) assert wilHab.pre in camKvy.kevers # Cam rotation with witness rotation @@ -214,7 +219,7 @@ def test_indexed_witness_replay(): rctMsgs = [] # list of receipt msgs from each witness for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 2 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -224,7 +229,7 @@ def test_indexed_witness_replay(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -235,7 +240,7 @@ def test_indexed_witness_replay(): dgkey = dbing.dgKey(pre=camHab.pre, dig=camHab.kever.serder.said) wigs = camHab.db.getWigs(dgkey) assert len(wigs) == 3 - wigers = [coring.Siger(qb64b=bytes(wig)) for wig in wigs] + wigers = [indexing.Siger(qb64b=bytes(wig)) for wig in wigs] rserder = eventing.receipt(pre=camHab.pre, sn=camHab.kever.sn, said=camHab.kever.serder.said) @@ -243,7 +248,7 @@ def test_indexed_witness_replay(): assert len(camRotWitRctMsg) == 413 for i in range(len(camWitKvys)): kvy = camWitKvys[i] - parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=kvy, local=True) # kvy.process(ims=bytearray(camRotWitRctMsg)) # send copy of witness rcts assert len(kvy.db.getWigs(dgkey)) == 3 # fully witnessed assert len(kvy.cues) == 0 # no cues @@ -256,10 +261,10 @@ def test_indexed_witness_replay(): # assert vcKvr.wits == camHab.kever.wits # send Cam rot's witness rcts to Van first then send Cam rot - parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camRotWitRctMsg), kvy=vanKvy, local=True) vanKvy.processEscrows() assert vcKvr.sn == 1 - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy, local=True) assert vcKvr.sn == 1 vanKvy.processEscrows() assert vcKvr.sn == 2 @@ -291,7 +296,7 @@ def test_nonindexed_witness_receipts(): wam is a witness """ - salt = coring.Salter(raw=b'abcdef0123456789').qb64 + salt = core.Salter(raw=b'abcdef0123456789').qb64 with habbing.openHby(name="cam", base="test", salt=salt) as camHby, \ habbing.openHby(name="van", base="test", salt=salt) as vanHby, \ @@ -361,7 +366,7 @@ def test_nonindexed_witness_receipts(): camIcpMsg = camHab.makeOwnInception() rctMsgs = [] # list of receipts from each witness for i, kvy in enumerate(camWitKvys): - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=kvy, local=True) # accepted event with cam sigs since own witness assert kvy.kevers[camHab.pre].sn == 0 assert len(kvy.cues) >= 1 # at least queued receipt cue @@ -371,7 +376,7 @@ def test_nonindexed_witness_receipts(): rctMsgs.append(rctMsg) for msg in rctMsgs: # Cam process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -382,7 +387,7 @@ def test_nonindexed_witness_receipts(): snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) # Van process rct msgs from all witnesses for Cam's icp message for i, msg in enumerate(rctMsgs): - parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) # escrows to Ure assert vanKvy.db.cntUres(snkey) == i + 1 # escrows assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow @@ -398,7 +403,7 @@ def test_nonindexed_witness_receipts(): assert camHab.pre not in vanKvy.kevers # still not accepted # Van process icp message from Cam - parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIcpMsg), kvy=vanKvy, local=True) # event accepted in database with sigs but not into KEL assert vanKvy.db.cntSigs(dgkey) == len(camHab.kever.verfers) assert vanKvy.db.cntPwes(snkey) == 1 # now in partial witness escrow @@ -420,7 +425,7 @@ def test_nonindexed_witness_receipts(): camIxnMsg = camHab.interact() rctMsgs = [] # list of receipts from each witness for i, kvy in enumerate(camWitKvys): - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=kvy, local=True) # kvy.process(ims=bytearray(camIxnMsg)) # send copy of cam icp msg to witness assert kvy.kevers[camHab.pre].sn == 1 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue @@ -431,7 +436,7 @@ def test_nonindexed_witness_receipts(): rctMsgs.append(rctMsg) for msg in rctMsgs: # Cam process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -442,7 +447,7 @@ def test_nonindexed_witness_receipts(): snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) # Van process rct msgs from all witnesses for Cam's ixn message for i, msg in enumerate(rctMsgs): - parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) # escrows to Ure assert vanKvy.db.cntUres(snkey) == i + 1 # escrows assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow @@ -457,7 +462,7 @@ def test_nonindexed_witness_receipts(): assert vcKvr.wits == wits # no change # Van process ixn message from Cam - parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camIxnMsg), kvy=vanKvy, local=True) # event accepted in database with sigs but not into KEL assert vanKvy.db.cntSigs(dgkey) == len(camHab.kever.verfers) assert vanKvy.db.cntPwes(snkey) == 1 # now in partial witness escrow @@ -477,14 +482,14 @@ def test_nonindexed_witness_receipts(): # Cam update itself with Wil receipts including Wils inception camReplayMsg = camHab.replay() assert len(camReplayMsg) == 2038 - parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy) + parsing.Parser().parse(ims=bytearray(camReplayMsg), kvy=wilKvy, local=True) assert camHab.pre in wilKvy.kevers assert wilKvy.kevers[camHab.pre].sn == 1 # asscepted both events assert len(wilKvy.cues) == 2 wilRctMsg = wilHab.processCues(wilKvy.cues) # process cue returns rct msg assert len(wilKvy.cues) == 0 - parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=camKvy, local=True) assert wilHab.pre in camKvy.kevers # Cam rotation with witness rotation @@ -503,7 +508,7 @@ def test_nonindexed_witness_receipts(): rctMsgs = [] # list of receipt msgs from each witness for i, kvy in enumerate(camWitKvys): - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=kvy, local=True) assert kvy.kevers[camHab.pre].sn == 2 # accepted event assert len(kvy.cues) >= 1 # at least queued receipt cue # better to find receipt cue in cues exactly @@ -513,7 +518,7 @@ def test_nonindexed_witness_receipts(): rctMsgs.append(rctMsg) for msg in rctMsgs: # process rct msgs from all witnesses - parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=camKvy, local=True) for hab in camWitHabs: assert hab.pre in camKvy.kevers @@ -524,7 +529,7 @@ def test_nonindexed_witness_receipts(): snkey = dbing.snKey(pre=camHab.pre, sn=camHab.kever.serder.sn) # Van process rct msgs from all witnesses for Cam's ixn message for i, msg in enumerate(rctMsgs): - parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=vanKvy, local=True) # escrows to Ure assert vanKvy.db.cntUres(snkey) == i + 1 # escrows assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # all in escrow @@ -532,7 +537,7 @@ def test_nonindexed_witness_receipts(): assert vcKvr.sn == 1 # not rot yet # send stale receipts from Wil to Van - parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(wilRctMsg), kvy=vanKvy, local=True) assert vanKvy.db.cntUres(snkey) == len(rctMsgs) # no change @@ -544,7 +549,7 @@ def test_nonindexed_witness_receipts(): assert vcKvr.wits == oldwits # no change # Van process rot message from Cam - parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy) + parsing.Parser().parse(ims=bytearray(camRotMsg), kvy=vanKvy, local=True) # event accepted in database with sigs but not into KEL assert vanKvy.db.cntSigs(dgkey) == len(camHab.kever.verfers) assert vanKvy.db.cntPwes(snkey) == 1 # now in partial witness escrow @@ -576,9 +581,10 @@ def test_out_of_order_witnessed_events(): # Wes is his witness # Bam is verifying the key state for Bob from Wes - with habbing.openHby(name="wes", base="test") as wesHby, \ - habbing.openHby(name="bob", base="test") as bobHby, \ - habbing.openHby(name="bam", base="test") as bamHby: + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + with habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, \ + habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, \ + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby: # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name='wes', isith='1', icount=1, transferable=False) @@ -590,7 +596,7 @@ def test_out_of_order_witnessed_events(): # Create Bob's icp, pass to Wes and generate receipt. wesKvy = eventing.Kevery(db=wesHby.db, lax=False, local=False) bobIcp = bobHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(bobIcp), kvy=wesKvy, local=True) assert bobHab.pre in wesHab.kevers iserder = serdering.SerderKERI(raw=bytearray(bobIcp)) wesHab.receipt(serder=iserder) @@ -598,7 +604,7 @@ def test_out_of_order_witnessed_events(): # Rotate and get Bob's rot, pass to Wes and generate receipt. bobHab.rotate() bobRotMsg = bobHab.makeOwnEvent(sn=1) - parsing.Parser().parse(ims=bytearray(bobRotMsg), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(bobRotMsg), kvy=wesKvy, local=True) assert wesKvy.kevers[bobHab.pre].sn == 1 bobRot = serdering.SerderKERI(raw=bobRotMsg) wesHab.receipt(serder=bobRot) @@ -609,7 +615,7 @@ def test_out_of_order_witnessed_events(): msgs.extend(msg) bamKvy = eventing.Kevery(db=bamHby.db, lax=False, local=False) - parsing.Parser().parse(ims=msgs, kvy=bamKvy) + parsing.Parser().parse(ims=msgs, kvy=bamKvy, local=True) # Ensure the rot ended up in out-of-order escrow assert bobHab.pre not in bamKvy.kevers diff --git a/tests/db/test_baser_mapsize.py b/tests/db/test_baser_mapsize.py new file mode 100644 index 000000000..3c23d6bb8 --- /dev/null +++ b/tests/db/test_baser_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.db.test_baser_mapsize module + +""" +import os +import pytest + + +def test_baser_specific_env_var(): + from keri.db import basing + + os.environ['KERI_BASER_MAP_SIZE'] = '150000000' + + try: + baser = basing.Baser(name='test_baser', temp=True) + assert baser.MapSize == 150000000 + baser.close() + finally: + os.environ.pop('KERI_BASER_MAP_SIZE', None) + + +def test_baser_general_env_var_fallback(): + from keri.db import basing + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + baser = basing.Baser(name='test_baser_general', temp=True) + assert baser.MapSize == 250000000 + baser.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_baser_specific_takes_precedence(): + from keri.db import basing + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_BASER_MAP_SIZE'] = '200000000' + + try: + baser = basing.Baser(name='test_baser_precedence', temp=True) + assert baser.MapSize == 200000000 + baser.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_BASER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_baser_specific_env_var() + test_baser_general_env_var_fallback() + test_baser_specific_takes_precedence() diff --git a/tests/db/test_basing.py b/tests/db/test_basing.py index de76c1ecb..1db881361 100644 --- a/tests/db/test_basing.py +++ b/tests/db/test_basing.py @@ -7,25 +7,28 @@ import os from dataclasses import dataclass, asdict -import lmdb import pytest -from hio.base import doing -from keri.core.serdering import Serder -from tests.app import openMultiSig -from keri.kering import Versionage +import lmdb +from hio.base import doing +from keri import core from keri.app import habbing from keri.core import coring, eventing, serdering -from keri.core.coring import MtrDex -from keri.core.coring import Serials, versify -from keri.core.coring import Salter +from keri.core.coring import Kinds, versify, Seqner from keri.core.eventing import incept, rotate, interact, Kever +from keri.core.serdering import Serder from keri.db import basing from keri.db import dbing +from keri.db import subing from keri.db.basing import openDB, Baser, KeyStateRecord, OobiRecord from keri.db.dbing import (dgKey, onKey, snKey) from keri.db.dbing import openLMDB from keri.help.helping import datify, dictify +# this breaks when running as __main__ better to do a custom import call to +# walk the directory tree and import explicity rather than depend on it +# being a known package. Works with pytest because pytest contructs a path +# its test runner and imports the tests explicity +from tests.app import openMultiSig # this breaks when running as __main__ def test_baser(): @@ -117,7 +120,7 @@ def test_baser(): preb = 'DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8") digb = 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8") sn = 3 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) assert vs == 'KERI10JSON000014_' ked = dict(vs=vs, pre=preb.decode("utf-8"), @@ -159,6 +162,24 @@ def test_baser(): assert db.delEvt(key) == True assert db.getEvt(key) == None + # test eventsourcerecords .srcs + record = basing.EventSourceRecord() + assert db.esrs.get(key) == None + assert db.esrs.put(key, record) == True + actual = db.esrs.get(key) + assert actual == record + record.local = False + # put does not overwrite must pin + assert db.esrs.put(key, record) == False + actual = db.esrs.get(key) + assert actual.local != record.local + assert actual != record + assert not db.esrs.get(key) == record + assert db.esrs.pin(key, record) == True + actual = db.esrs.get(key) + assert actual.local == record.local + assert db.esrs.get(key) == record + # test first seen event log .fels sub db preA = b'BAKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' @@ -253,11 +274,11 @@ def test_baser(): # replay preB events in database items = [item for item in db.getFelItemPreIter(preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 items = [item for item in db.getFelItemPreIter(preB, fn=3)] - assert items == [(3, digX), (4, digY)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 items = [item for item in db.getFelItemPreIter(preB, fn=5)] @@ -462,100 +483,27 @@ def test_baser(): assert db.putUres(key=cKey, vals=cVals) assert db.putUres(key=dKey, vals=dVals) - # Test getUreItemsNext( key=b"") - # aVals - items = db.getUreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUreItemsNextIter()] + items = [item for item in db.getUreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals + vals = [bytes(val) for key, val in items] + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUreItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getUreItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - items = [item for item in db.getUreItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getUreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delUre(ikey, val) == True # bVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -565,7 +513,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # cVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -575,7 +523,7 @@ def test_baser(): assert db.delUre(ikey, val) == True # dVals - items = [item for item in db.getUreItemsNextIter(key=ikey)] + items = [item for item in db.getUreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -584,10 +532,6 @@ def test_baser(): for key, val in items: assert db.delUre(ikey, val) == True - # none - items = [item for item in db.getUreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Validator (transferable) Receipts # test .vrcs sub db methods dgkey @@ -669,100 +613,26 @@ def test_baser(): assert db.putVres(key=cKey, vals=cVals) assert db.putVres(key=dKey, vals=dVals) - # Test getVreItemsNext( key=b"") - # aVals - items = db.getVreItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getVreItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getVreItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getVreItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getVreItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getVreItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getVreItemsNextIter()] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getVreItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getVreItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getVreItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getVreItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getVreItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delVre(ikey, val) == True # bVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -772,7 +642,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # cVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -782,7 +652,7 @@ def test_baser(): assert db.delVre(ikey, val) == True # dVals - items = [item for item in db.getVreItemsNextIter(key=ikey)] + items = [item for item in db.getVreItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -791,10 +661,6 @@ def test_baser(): for key, val in items: assert db.delVre(ikey, val) == True - # none - items = [item for item in db.getVreItemsNext(key=ikey)] - assert items == [] # empty - assert not items # test .kels insertion order dup methods. dup vals are insertion order @@ -854,100 +720,28 @@ def test_baser(): assert db.putPses(key=cKey, vals=cVals) assert db.putPses(key=dKey, vals=dVals) - # Test getPseItemsNext( key=b"") - # aVals - items = db.getPseItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPseItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPseItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPseItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPseItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPseItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPseItemsNextIter()] + items = [item for item in db.getPseItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPseItemsNextIter(key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - items = [item for item in db.getPseItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getPseItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getPseItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delPse(ikey, val) == True + # bVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -957,7 +751,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # cVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -967,7 +761,7 @@ def test_baser(): assert db.delPse(ikey, val) == True # dVals - items = [item for item in db.getPseItemsNextIter(key=ikey)] + items = [item for item in db.getPseItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -976,33 +770,45 @@ def test_baser(): for key, val in items: assert db.delPse(ikey, val) == True - # none - items = [item for item in db.getPseItemsNext(key=ikey)] - assert items == [] # empty - assert not items - # Test .pdes partial delegated escrow seal source couples + # Test .udes partial delegated escrow seal source couples key = dgKey(preb, digb) assert key == f'{preb.decode("utf-8")}.{digb.decode("utf-8")}'.encode("utf-8") - # test .pdes sub db methods + # test .pdes SerderIoSetSuber methods + assert isinstance(db.pdes, subing.OnIoDupSuber) + + + # test .udes CatCesrSuber sub db methods + assert isinstance(db.udes, subing.CatCesrSuber) + assert db.udes.klas == (coring.Seqner, coring.Saider) + ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' ssnu2 = b'0AAAAAAAAAAAAAAAAAAAAAAC' sdig2 = b'EBYYJRCCpAGO7WjjsLhtHVR37Pawv67kveIFUPvt38x0' val1 = ssnu1 + sdig1 + tuple1 = (coring.Seqner(qb64b=ssnu1), coring.Saider(qb64b=sdig1)) val2 = ssnu2 + sdig2 + tuple2 = (coring.Seqner(qb64b=ssnu2), coring.Saider(qb64b=sdig2)) + + + assert db.udes.get(keys=key) == None + assert db.udes.rem(keys=key) == False + assert db.udes.put(keys=key, val=tuple1) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.put(keys=key, val=tuple2) == False + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val1 + assert db.udes.pin(keys=key, val=tuple2) == True + seqner, saider = db.udes.get(keys=key) + assert seqner.qb64b + saider.qb64b == val2 + assert db.udes.rem(keys=key) == True + assert db.udes.get(keys=key) == None + + - assert db.getPde(key) == None - assert db.delPde(key) == False - assert db.putPde(key, val1) == True - assert db.getPde(key) == val1 - assert db.putPde(key, val2) == False - assert db.getPde(key) == val1 - assert db.setPde(key, val2) == True - assert db.getPde(key) == val2 - assert db.delPde(key) == True - assert db.getPde(key) == None # Partially Witnessed Escrow Events # test .pwes insertion order dup methods. dup vals are insertion order @@ -1041,100 +847,26 @@ def test_baser(): assert db.putPwes(key=cKey, vals=cVals) assert db.putPwes(key=dKey, vals=dVals) - # Test getPweItemsNext( key=b"") - # aVals - items = db.getPweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getPweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getPweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getPweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getPweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getPweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getPweItemsNextIter()] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPweItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getPweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getPweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getPweItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getPweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delPwe(ikey, val) == True # bVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1144,7 +876,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # cVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1154,7 +886,7 @@ def test_baser(): assert db.delPwe(ikey, val) == True # dVals - items = [item for item in db.getPweItemsNextIter(key=ikey)] + items = [item for item in db.getPweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1163,10 +895,6 @@ def test_baser(): for key, val in items: assert db.delPwe(ikey, val) == True - # none - items = [item for item in db.getPweItemsNext(key=ikey)] - assert items == [] # empty - assert not items # Unverified Witness Receipt Escrows # test .uwes insertion order dup methods. dup vals are insertion order @@ -1205,100 +933,26 @@ def test_baser(): assert db.putUwes(key=cKey, vals=cVals) assert db.putUwes(key=dKey, vals=dVals) - # Test getUweItemsNext( key=b"") - # aVals - items = db.getUweItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getUweItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getUweItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getUweItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getUweItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getUweItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getUweItemsNextIter()] + items = [item for item in db.getUweItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getUweItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getUweItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - items = [item for item in db.getUweItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in db.getUweItemsNextIter(key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert db.delUwe(ikey, val) == True - # bVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1308,7 +962,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # cVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1318,7 +972,7 @@ def test_baser(): assert db.delUwe(ikey, val) == True # dVals - items = [item for item in db.getUweItemsNextIter(key=ikey)] + items = [item for item in db.getUweItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1327,10 +981,7 @@ def test_baser(): for key, val in items: assert db.delUwe(ikey, val) == True - # none - items = [item for item in db.getUweItemsNext(key=ikey)] - assert items == [] # empty - assert not items + # test .ooes insertion order dup methods. dup vals are insertion order key = b'A' @@ -1367,100 +1018,25 @@ def test_baser(): assert db.putOoes(key=cKey, vals=cVals) assert db.putOoes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getOoeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getOoeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getOoeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getOoeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getOoeItemsNext(key=ikey) - assert items == [] # empty - assert not items - # Test getOoeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getOoeItemsNextIter()] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getOoeItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getOoeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getOoeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getOoeItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getOoeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delOoe(ikey, val) == True # bVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1470,7 +1046,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # cVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1480,7 +1056,7 @@ def test_baser(): assert db.delOoe(ikey, val) == True # dVals - items = [item for item in db.getOoeItemsNextIter(key=ikey)] + items = [item for item in db.getOoeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1489,12 +1065,6 @@ def test_baser(): for key, val in items: assert db.delOoe(ikey, val) == True - # none - items = [item for item in db.getOoeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - # test .dels insertion order dup methods. dup vals are insertion order key = b'A' vals = [b"z", b"m", b"x", b"a"] @@ -1547,100 +1117,26 @@ def test_baser(): assert db.putLdes(key=cKey, vals=cVals) assert db.putLdes(key=dKey, vals=dVals) - # Test getOoeItemsNext( key=b"") - # aVals - items = db.getLdeItemsNext() # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = db.getLdeItemsNext(key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = db.getLdeItemsNext(key=b'', skip=False) # get frist key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = db.getLdeItemsNext(key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = db.getLdeItemsNext(key=ikey) - assert items == [] # empty - assert not items # Test getLdeItemsNextIter(key=b"") # get dups at first key in database # aVals - items = [item for item in db.getLdeItemsNextIter()] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getLdeItemsNextIter(key=aKey, skip=False)] + items = [item for item in db.getLdeItemIter()] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in db.getLdeItemsNextIter(key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals + assert vals == aVals + bVals + cVals + dVals - items = [item for item in db.getLdeItemsNextIter(key=b'', skip=False)] + items = [item for item in db.getLdeItemIter(key=aKey)] assert items # not empty ikey = items[0][0] assert ikey == aKey vals = [val for key, val in items] assert vals == aVals - for key, val in items: - assert db.delLde(ikey, val) == True # bVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=bKey)] assert items # not empty ikey = items[0][0] assert ikey == bKey @@ -1650,7 +1146,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # cVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=cKey)] assert items # not empty ikey = items[0][0] assert ikey == cKey @@ -1660,7 +1156,7 @@ def test_baser(): assert db.delLde(ikey, val) == True # dVals - items = [item for item in db.getLdeItemsNextIter(key=ikey)] + items = [item for item in db.getLdeItemIter(key=dKey)] assert items # not empty ikey = items[0][0] assert ikey == dKey @@ -1669,13 +1165,6 @@ def test_baser(): for key, val in items: assert db.delLde(ikey, val) == True - # none - items = [item for item in db.getLdeItemsNext(key=ikey)] - assert items == [] # empty - assert not items - - - assert not os.path.exists(db.path) """ End Test """ @@ -1687,8 +1176,8 @@ def test_clean_baser(): """ name = "nat" # with basing.openDB(name="nat") as natDB, keeping.openKS(name="nat") as natKS: - with habbing.openHby(name=name) as hby: # default is temp=True - natHab = hby.makeHab(name=name, isith='2', icount=3) + with habbing.openHby(name=name, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # default is temp=True + natHab = hby.makeHab(name=name, isith='2', icount=3) # default Hab # setup Nat's habitat using default salt multisig already incepts #natHab = habbing.Habitat(name='nat', ks=natKS, db=natDB, #isith='2', icount=3, temp=True) @@ -1733,7 +1222,7 @@ def test_clean_baser(): assert natHab.db.env.stat()['entries'] <= 96 #68 # verify name pre kom in db - data = natHab.db.habs.get(keys=natHab.name) + data = natHab.db.habs.get(keys=natHab.pre) assert data.hid == natHab.pre # add garbage event to corrupt database @@ -1792,7 +1281,7 @@ def test_clean_baser(): assert state.f == '6' # verify name pre kom in db - data = natHab.db.habs.get(keys=natHab.name) + data = natHab.db.habs.get(keys=natHab.pre) assert data.hid == natHab.pre @@ -1810,7 +1299,7 @@ def test_fetchkeldel(): preb = 'BWzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8") digb = 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8") sn = 3 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) assert vs == 'KERI10JSON000014_' ked = dict(vs=vs, pre=preb.decode("utf-8"), @@ -1882,7 +1371,7 @@ def test_fetchkeldel(): assert vals == lastvals - # test getDelIter + # test getDelItemIter preb = 'BTmuupUhPx5_yZ-Wk1x4ejhccWzwEHHzq7K0gzQPYGGw'.encode("utf-8") sn = 1 # do not start at zero key = snKey(preb, sn) @@ -1903,8 +1392,8 @@ def test_fetchkeldel(): for val in vals2: assert db.addDe(key, val) == True - vals = [bytes(val) for val in db.getDelIter(preb)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in db.getDelItemIter(preb)] assert vals == allvals assert not os.path.exists(db.path) @@ -1917,7 +1406,7 @@ def test_usebaser(): Test using Baser """ raw = b'g\x15\x89\x1a@\xa4\xa47\x07\xb9Q\xb8\x18\xcdJW' - salter = Salter(raw=raw) + salter = core.Salter(raw=raw) # create coe's signers signers = salter.signers(count=8, path='db', temp=True) @@ -1929,7 +1418,7 @@ def test_usebaser(): count = len(keys) nxtkeys = [signers[3].verfer.qb64b, signers[4].verfer.qb64b, signers[5].verfer.qb64b] sith = "2" - code = MtrDex.Blake3_256 # Blake3 digest of incepting data + code = core.MtrDex.Blake3_256 # Blake3 digest of incepting data serder = incept(keys=keys, code=code, isith=sith, @@ -2068,6 +1557,40 @@ def test_keystaterecord(): assert nksr._asdict() == ksn + """End Test""" + +def test_eventsourcerecord(): + """ + Test EventSourceRecord dataclass + """ + record = basing.EventSourceRecord() # default local is True + assert isinstance(record, basing.EventSourceRecord) + assert record.local is True + assert record.local + assert "local" in record # asdict means in is against the keys (labels) + assert (asdict(record)) == {'local': True} + + record.local = False + assert record.local is False + assert not record.local + assert (asdict(record)) == {'local': False} + + record = basing.EventSourceRecord(local=False) + assert isinstance(record, basing.EventSourceRecord) + assert record.local is False + assert not record.local + assert "local" in record # asdict means in is against the keys (labels) + assert (asdict(record)) == {'local': False} + + record = basing.EventSourceRecord(local=None) + assert isinstance(record, basing.EventSourceRecord) + assert record.local is None + assert not record.local + assert "local" in record # asdict means in is against the keys (labels) + assert (asdict(record)) == {'local': None} + + + """End Test""" @@ -2075,12 +1598,14 @@ def test_dbdict(): """ Test custom dbdict subclass of dict """ - dbd = basing.dbdict(a=1, b=2, c=3) + dbd = basing.dbdict(a=1, b=2, c=3) # init in memory so never acesses db assert dbd.db == None assert 'a' in dbd assert 'b' in dbd assert 'c' in dbd assert [(k, v) for k, v in dbd.items()] == [('a', 1), ('b', 2), ('c', 3)] + assert list(dbd.keys()) == ['a', 'b', 'c'] + assert list(dbd.values()) == [1, 2, 3] assert dbd.get('a') == 1 assert dbd['a'] == 1 @@ -2293,7 +1818,7 @@ def test_KERI_BASER_MAP_SIZE_handles_bad_values(caplog): def test_clear_escrows(): with openDB() as db: - key = b'A.a' + key = b'A' vals = [b"z", b"m", b"x", b"a"] db.putUres(key, vals) @@ -2303,43 +1828,59 @@ def test_clear_escrows(): db.putUwes(key, vals) db.putOoes(key, vals) db.putLdes(key, vals) - db.putQnfs(key, vals) - preb = 'DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8") - digb = 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8") - key = dgKey(preb, digb) - ssnu1 = b'0AAAAAAAAAAAAAAAAAAAAAAB' - sdig1 = b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E' - val1 = ssnu1 + sdig1 + pre = b'k' + snh = b'snh' + saidb = b'saidb' + db.qnfs.add(keys=(pre, saidb), val=b"z") + assert db.qnfs.cnt(keys=(pre, saidb)) == 1 - db.putPde(key, val1) + db.misfits.add(keys=(pre, snh), val=saidb) + assert db.misfits.cnt(keys=(pre, snh)) == 1 + + db.delegables.add(snKey(pre, 0), saidb) + assert db.delegables.cnt(keys=snKey(pre, 0)) == 1 + + db.pdes.addOn(keys=pre, on=0, val=saidb) + assert db.pdes.cnt(keys=snKey(pre, 0)) == 1 + + udesKey = dgKey('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc'.encode("utf-8"), + 'EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4'.encode("utf-8")) + db.udes.put(keys=udesKey, val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), + coring.Saider(qb64b=b'EALkveIFUPvt38xhtgYYJRCCpAGO7WjjHVR37Pawv67E'))) + assert db.udes.get(keys=udesKey) is not None - pre = 'k' saider = coring.Saider(qb64b='EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') db.rpes.put(keys=('route',), vals=[saider]) - assert db.rpes.cntAll() == 1 + assert db.rpes.cnt(keys=('route',)) == 1 + + db.epsd.put(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',), val=coring.Dater()) + assert db.epsd.get(keys=('DAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc',)) is not None db.eoobi.pin(keys=('url',), val=OobiRecord()) assert db.eoobi.cntAll() == 1 + serder = Serder(raw=b'{"v":"KERI10JSON0000cb_","t":"ixn","d":"EG8WAmM29ZBdoXbnb87yiPxQw4Y7gcQjqZS74vBAKsRm","i":"DApYGFaqnrALTyejaJaGAVhNpSCtqyerPqWVK9ZBNZk0","s":"4","p":"EAskHI462CuIMS_gNkcl_QewzrRSKH2p9zHQIO132Z30","a":[]}') + db.dpub.put(keys=(pre, 'said'), val=serder) + assert db.dpub.get(keys=(pre, 'said')) is not None + db.gpwe.add(keys=(pre,), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) - assert db.gpwe.cntAll() == 1 + assert db.gpwe.cnt(keys=(pre,)) == 1 db.gdee.add(keys=(pre,), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) - assert db.gdee.cntAll() == 1 + assert db.gdee.cnt(keys=(pre,)) == 1 - serder = Serder(raw=b'{"v":"KERI10JSON0000cb_","t":"ixn","d":"EG8WAmM29ZBdoXbnb87yiPxQw4Y7gcQjqZS74vBAKsRm","i":"DApYGFaqnrALTyejaJaGAVhNpSCtqyerPqWVK9ZBNZk0","s":"4","p":"EAskHI462CuIMS_gNkcl_QewzrRSKH2p9zHQIO132Z30","a":[]}') db.dpwe.pin(keys=(pre, 'said'), val=serder) - assert db.dpwe.cntAll() == 1 + assert db.dpwe.get(keys=(pre, 'said')) is not None db.gpse.add(keys=('qb64',), val=(coring.Seqner(qb64b=b'0AAAAAAAAAAAAAAAAAAAAAAB'), saider)) - assert db.gpse.cntAll() == 1 + assert db.gpse.cnt(keys=('qb64',)) == 1 db.epse.put(keys=('dig',), val=serder) - assert db.epse.cntAll() == 1 + assert db.epse.get(keys=('dig',)) is not None db.dune.pin(keys=(pre, 'said'), val=serder) - assert db.dune.cntAll() == 1 + assert db.dune.get(keys=(pre, 'said')) is not None db.clearEscrows() @@ -2350,8 +1891,8 @@ def test_clear_escrows(): assert db.getUwes(key) == [] assert db.getOoes(key) == [] assert db.getLdes(key) == [] - assert db.getQnfs(key) == [] - assert db.getPdes(key) == [] + assert db.qnfs.cntAll() == 0 + assert db.pdes.cntAll() == 0 assert db.rpes.cntAll() == 0 assert db.eoobi.cntAll() == 0 assert db.gpwe.cntAll() == 0 @@ -2360,6 +1901,11 @@ def test_clear_escrows(): assert db.gpse.cntAll() == 0 assert db.epse.cntAll() == 0 assert db.dune.cntAll() == 0 + assert db.misfits.cntAll() == 0 + assert db.delegables.cntAll() == 0 + assert db.udes.cntAll() == 0 + assert db.epsd.cntAll() == 0 + assert db.dpub.cntAll() == 0 if __name__ == "__main__": test_baser() diff --git a/tests/db/test_dbing.py b/tests/db/test_dbing.py index 56c1fa40a..309a366df 100644 --- a/tests/db/test_dbing.py +++ b/tests/db/test_dbing.py @@ -17,7 +17,7 @@ from keri.db import dbing from keri.db.dbing import clearDatabaserDir, openLMDB from keri.db.dbing import (dgKey, onKey, fnKey, snKey, dtKey, splitKey, - splitKeyON, splitKeyFN, splitKeySN, splitKeyDT) + splitOnKey, splitKeyFN, splitSnKey, splitKeyDT) from keri.db.dbing import LMDBer @@ -35,11 +35,41 @@ def test_key_funcs(): sn = 3 dts = b'2021-02-13T19:16:50.750302+00:00' + + # test onKey generator of key from top key and trailing ordinal number + assert onKey(pre, 0) == pre + b'.' + b"%032x" % 0 + assert onKey(pre, 1) == pre + b'.' + b"%032x" % 1 + assert onKey(pre, 2) == pre + b'.' + b"%032x" % 2 + assert onKey(pre, 3) == pre + b'.' + b"%032x" % 3 + assert onKey(pre, 4) == pre + b'.' + b"%032x" % 4 + + assert onKey(pre, 0, sep=b'|') == pre + b'|' + b"%032x" % 0 + assert onKey(pre, 4, sep=b'|') == pre + b'|' + b"%032x" % 4 + + assert (onkey := onKey(top=pre, on=0)) == pre + b'.' + b"%032x" % 0 + assert splitKey(key=onkey) == (pre, b"%032x" % 0) + assert splitOnKey(onkey) == (pre, 0) + assert (onkey := onKey(top=pre, on=1)) == pre + b'.' + b"%032x" % 1 + assert splitKey(key=onkey) == (pre, b"%032x" % 1) + assert splitOnKey(onkey) == (pre, 1) + assert (onkey := onKey(top=pre, on=15)) == pre + b'.' + b"%032x" % 15 + assert splitKey(key=onkey) == (pre, b"%032x" % 15) + assert splitOnKey(onkey) == (pre, 15) + + assert (onkey := onKey(top=pre, on=0, sep=b'|')) == pre + b'|' + b"%032x" % 0 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 0) + assert splitOnKey(onkey, sep=b'|') == (pre, 0) + assert (onkey := onKey(top=pre, on=15, sep=b'|')) == pre + b'|' + b"%032x" % 15 + assert splitKey(key=onkey, sep=b'|') == (pre, b"%032x" % 15) + assert splitOnKey(onkey, sep=b'|') == (pre, 15) + + + # test snKey assert snKey(pre, sn) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn)) == (pre, b'%032x' % sn) - assert splitKeySN(snKey(pre, sn)) == (pre, sn) + assert splitSnKey(snKey(pre, sn)) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -63,7 +93,7 @@ def test_key_funcs(): b'.00000000000000000000000000000003') assert splitKey(snKey(pre, sn).decode("utf-8")) == (pre, '%032x' % sn) - assert splitKeySN(snKey(pre, sn).decode("utf-8")) == (pre, sn) + assert splitSnKey(snKey(pre, sn).decode("utf-8")) == (pre, sn) assert dgKey(pre, dig) == (b'BAzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc' b'.EGAPkzNZMtX-QiVgbRbyAIZGoXvbGv9IPb0foWTZvI_4') @@ -83,8 +113,7 @@ def test_key_funcs(): with pytest.raises(ValueError): splitKey(pre) - with pytest.raises(ValueError): - splitKey(dgKey(pre, dgKey(pre, dig))) + assert splitKey(dgKey(pre, dgKey(pre, dig))) # rsplit on gets trailing part # memoryview # Bytes @@ -95,7 +124,7 @@ def test_key_funcs(): key = memoryview(snKey(pre, sn)) assert splitKey(key) == (pre, b'%032x' % sn) - assert splitKeySN(key) == (pre, sn) + assert splitSnKey(key) == (pre, sn) key = memoryview(dgKey(pre, dig)) assert splitKey(key) == (pre, dig) @@ -227,15 +256,8 @@ def test_lmdber(): assert databaser.path == None assert databaser.env == None - os.environ['KERI_LMDB_MAP_SIZE'] = f'{2 * 1024**3}' # 2GB databaser.reopen() assert databaser.opened - assert databaser.env.info()['map_size'] == 2 * 1024**3 # 2GB - - os.environ['KERI_LMDB_MAP_SIZE'] = f'invalid-size' # will trigger default - databaser.reopen() - assert databaser.opened - assert databaser.env.info()['map_size'] == 4 * 1024 ** 3 # 4GB default value assert isinstance(databaser.env, lmdb.Environment) assert databaser.path.endswith("keri/db/main") assert databaser.env.path() == databaser.path @@ -270,7 +292,7 @@ def test_lmdber(): assert dber.delVal(db, key) == True assert dber.getVal(db, key) == None - # Test getAllItemIter(self, db, key=b'', split=True, sep=b'.') + # Test getTopItemIter key = b"a.1" val = b"wow" assert dber.putVal(db, key, val) == True @@ -280,21 +302,22 @@ def test_lmdber(): key = b"b.1" val = b"woo" assert dber.putVal(db, key, val) == True - assert [(bytes(pre), bytes(num), bytes(val)) for pre, num, val - in dber.getAllItemIter(db=db)] == [(b'a', b'1', b'wow'), - (b'a', b'2', b'wee'), - (b'b', b'1', b'woo')] + assert [(bytes(key), bytes(val)) for key, val + in dber.getTopItemIter(db=db)] == [(b'a.1', b'wow'), + (b'a.2', b'wee'), + (b'b.1', b'woo')] - assert dber.delTopVal(db, key=b"a.") + assert dber.delTopVal(db, top=b"a.") items = [ (key, bytes(val)) for key, val in dber.getTopItemIter(db=db )] assert items == [(b'b.1', b'woo')] - # test OrdVal OrdItem ordinal numbered event sub db + # test Ordinal Numbered ON keyed value methods db = dber.env.open_db(key=b'seen.') preA = b'BBKY1sKmgyjAiUDdUBPNPyrSz_ad_Qf9yzhDNZlEKiMc' preB = b'EH7Oq9oxCgYa-nnNLvwhp9sFZpALILlRYyB-6n4WDi7w' preC = b'EIDA1n-WiBA0A8YOqnKrB-wWQYYC49i5zY_qrIZIicQg' + preD = b'EAYC49i5zY_qrIZIicQgIDA1n-WiBA0A8YOqnKrB-wWQ' keyA0 = onKey(preA, 0) @@ -323,13 +346,23 @@ def test_lmdber(): assert dber.putVal(db, keyA0, val=digA) == False assert dber.setVal(db, keyA0, val=digA) == True assert dber.getVal(db, keyA0) == digA + assert dber.getOnVal(db, preA, 0) == digA assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None + assert dber.getOnVal(db, preA, 0) == None + + assert dber.putOnVal(db, preA, 0, val=digA) == True + assert dber.getOnVal(db, preA, 0) == digA + assert dber.putOnVal(db, preA, 0, val=digA) == False + assert dber.setOnVal(db, preA, 0, val=digA) == True + assert dber.getOnVal(db, preA, 0) == digA + assert dber.delOnVal(db, preA, 0) == True + assert dber.getOnVal(db, preA, 0) == None - # test appendOrdValPre + # test appendOnValPre # empty database assert dber.getVal(db, keyB0) == None - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -337,7 +370,7 @@ def test_lmdber(): # earlier pre in database only assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -346,7 +379,7 @@ def test_lmdber(): # earlier and later pre in db but not same pre assert dber.getVal(db, keyA0) == digA assert dber.putVal(db, keyC0, val=digC) == True - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU assert dber.delVal(db, keyB0) == True @@ -356,13 +389,13 @@ def test_lmdber(): assert dber.delVal(db, keyA0) == True assert dber.getVal(db, keyA0) == None assert dber.getVal(db, keyC0) == digC - on = dber.appendOrdValPre(db, preB, digU) + on = dber.appendOnVal(db, preB, digU) assert on == 0 assert dber.getVal(db, keyB0) == digU # earlier pre and later pre and earlier entry for same pre assert dber.putVal(db, keyA0, val=digA) == True - on = dber.appendOrdValPre(db, preB, digV) + on = dber.appendOnVal(db, preB, digV) assert on == 1 assert dber.getVal(db, keyB1) == digV @@ -372,51 +405,126 @@ def test_lmdber(): assert dber.delVal(db, keyC0) == True assert dber.getVal(db, keyC0) == None # another value for preB - on = dber.appendOrdValPre(db, preB, digW) + on = dber.appendOnVal(db, preB, digW) assert on == 2 assert dber.getVal(db, keyB2) == digW # yet another value for preB - on = dber.appendOrdValPre(db, preB, digX) + on = dber.appendOnVal(db, preB, digX) assert on == 3 assert dber.getVal(db, keyB3) == digX # yet another value for preB - on = dber.appendOrdValPre(db, preB, digY ) + on = dber.appendOnVal(db, preB, digY ) assert on == 4 assert dber.getVal(db, keyB4) == digY - assert dber.cntValsAllPre(db, preB) == 5 + assert dber.appendOnVal(db, preD, digY ) == 0 - # replay preB events in database - items = [item for item in dber.getAllOrdItemPreIter(db, preB)] - assert items == [(0, digU), (1, digV), (2, digW), (3, digX), (4, digY)] + assert dber.cntOnVals(db, key=preB) == 5 + assert dber.cntOnVals(db, key=b'') == 6 # all keys + assert dber.cntOnVals(db) == 6 # all keys + + # iter replay + # replay preB event items in database + items = [item for item in dber.getOnItemIter(db, preB)] + assert items == [(preB, 0, digU), (preB, 1, digV), (preB, 2, digW), + (preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 3 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=3)] - assert items == [(3, digX), (4, digY)] + items = [item for item in dber.getOnItemIter(db, preB, on=3)] + assert items == [(preB, 3, digX), (preB, 4, digY)] # resume replay preB events at on = 5 - items = [item for item in dber.getAllOrdItemPreIter(db, preB, on=5)] + items = [item for item in dber.getOnItemIter(db, preB, on=5)] assert items == [] # replay all events in database with pre events before and after assert dber.putVal(db, keyA0, val=digA) == True assert dber.putVal(db, keyC0, val=digC) == True - items = [item for item in dber.getAllOrdItemAllPreIter(db)] - assert items == [(preA, 0, digA), (preB, 0, digU), (preB, 1, digV), - (preB, 2, digW), (preB, 3, digX), (preB, 4, digY), + items = [item for item in dber.getOnItemIter(db, key=b'')] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), (preC, 0, digC)] + items = [item for item in dber.getOnItemIter(db)] + assert items == [(preA, 0, digA), + (preD, 0, digY), + (preB, 0, digU), + (preB, 1, digV), + (preB, 2, digW), + (preB, 3, digX), + (preB, 4, digY), + (preC, 0, digC)] # resume replay all starting at preB on=2 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=keyB2)] - assert items == [(preB, 2, digW), (preB, 3, digX), (preB, 4, digY), - (preC, 0, digC)] + top, on = splitOnKey(keyB2) + items = [item for item in dber.getOnItemIter(db, key=top, on=on)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] # resume replay all starting at preC on=1 - items = [item for item in dber.getAllOrdItemAllPreIter(db, key=onKey(preC, 1))] + items = [item for item in dber.getOnItemIter(db, key=preC, on=1)] assert items == [] + # val replay + # replay preB event vals in database + vals = [val for val in dber.getOnValIter(db, preB)] + assert vals == [digU, digV, digW, digX, digY] + + # resume replay preB events at on = 3 + vals = [val for val in dber.getOnValIter(db, preB, on=3)] + assert vals == [digX, digY] + + # resume replay preB events at on = 5 + vals = [val for val in dber.getOnValIter(db, preB, on=5)] + assert vals == [] + + vals = [val for val in dber.getOnValIter(db, key=b'')] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + vals = [val for val in dber.getOnValIter(db)] + assert vals == [digA, + digY, + digU, + digV, + digW, + digX, + digY, + digC] + + # resume replay all starting at preB on=2 + top, on = splitOnKey(keyB2) + vals = [val for val in dber.getOnValIter(db, key=top, on=on)] + assert vals == [digW, digX, digY] + + # resume replay all starting at preC on=1 + vals = [val for val in dber.getOnValIter(db, key=preC, on=1)] + assert vals == [] + + + # test delOnVal + assert dber.delOnVal(db, key=preB) # default on=0 + assert not dber.delOnVal(db, key=preB, on=0) + assert dber.delOnVal(db, key=preB, on=1) + assert not dber.delOnVal(db, key=preB, on=1) + + items = [item for item in dber.getOnItemIter(db, key=preB)] + assert items == [(top, 2, digW), (top, 3, digX), (top, 4, digY)] + + with pytest.raises(KeyError): + assert dber.delOnVal(db, key=b'') # empty key + # test Vals dup methods. dup vals are lexocographic key = b'A' @@ -452,104 +560,65 @@ def test_lmdber(): vals = [b"z", b"m", b"x", b"a"] db = dber.env.open_db(key=b'peep.', dupsort=True) - assert dber.getIoVals(db, key) == [] - assert dber.getIoValLast(db, key) == None - assert dber.cntIoVals(db, key) == 0 - assert dber.delIoVals(db, key) == False - assert dber.putIoVals(db, key, vals) == True - assert dber.getIoVals(db, key) == vals # preserved insertion order - assert dber.cntIoVals(db, key) == len(vals) == 4 - assert dber.getIoValLast(db, key) == vals[-1] - assert dber.putIoVals(db, key, vals=[b'a']) == False # duplicate - assert dber.getIoVals(db, key) == vals # no change - assert dber.addIoVal(db, key, val=b'b') == True - assert dber.addIoVal(db, key, val=b'a') == False - assert dber.getIoVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] - assert [val for val in dber.getIoValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] - assert dber.delIoVals(db, key) == True - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.getIoDupValLast(db, key) == None + assert dber.cntIoDupVals(db, key) == 0 + assert dber.delIoDupVals(db, key) == False + assert dber.putIoDupVals(db, key, vals) == True + assert dber.getIoDupVals(db, key) == vals # preserved insertion order + assert dber.cntIoDupVals(db, key) == len(vals) == 4 + assert dber.getIoDupValLast(db, key) == vals[-1] + assert dber.putIoDupVals(db, key, vals=[b'a']) == False # duplicate + assert dber.getIoDupVals(db, key) == vals # no change + assert dber.addIoDupVal(db, key, val=b'b') == True + assert dber.addIoDupVal(db, key, val=b'a') == False + assert dber.getIoDupVals(db, key) == [b"z", b"m", b"x", b"a", b"b"] + assert [val for val in dber.getIoDupValsIter(db, key)] == [b"z", b"m", b"x", b"a", b'b'] + assert dber.delIoDupVals(db, key) == True + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in vals: - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] - assert dber.putIoVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] + assert dber.putIoDupVals(db, key, vals) == True for val in sorted(vals): - assert dber.delIoVal(db, key, val) - assert dber.getIoVals(db, key) == [] + assert dber.delIoDupVal(db, key, val) + assert dber.getIoDupVals(db, key) == [] #delete and add in odd order - assert dber.putIoVals(db, key, vals) == True - assert dber.delIoVal(db, key, vals[2]) - assert dber.addIoVal(db, key, b'w') - assert dber.delIoVal(db, key, vals[0]) - assert dber.addIoVal(db, key, b'e') - assert dber.getIoVals(db, key) == [b'm', b'a', b'w', b'e'] - - # Test getIoValsAllPreIter(self, db, pre) - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True - - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals) == True + assert dber.delIoDupVal(db, key, vals[2]) + assert dber.addIoDupVal(db, key, b'w') + assert dber.delIoDupVal(db, key, vals[0]) + assert dber.addIoDupVal(db, key, b'e') + assert dber.getIoDupVals(db, key) == [b'm', b'a', b'w', b'e'] - vals = [bytes(val) for val in dber.getIoValsAllPreIter(db, pre)] - allvals = vals0 + vals1 + vals2 - assert vals == allvals - - # Test getIoValsLastAllPreIter(self, db, pre) - pre = b'BAejWzwQPYGGwTmuupUhPx5_yZ-Wk1xEHHzq7K0gzhcc' - vals0 = [b"gamma", b"beta"] - sn = 0 - key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True - vals1 = [b"mary", b"peter", b"john", b"paul"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True - - vals2 = [b"dog", b"cat", b"bird"] - sn += 1 - key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True - - vals = [bytes(val) for val in dber.getIoValLastAllPreIter(db, pre)] - lastvals = [vals0[-1], vals1[-1], vals2[-1]] - assert vals == lastvals - - # Test getIoValsAnyPreIter(self, db, pre) + # Test TopIoDupItemIter(self, db, pre) pre = b'BBPYGGwTmuupUhPx5_yZ-Wk1x4ejWzwEHHzq7K0gzhcc' vals0 = [b"gamma", b"beta"] sn = 1 # not start at zero key = snKey(pre, sn) - assert dber.addIoVal(db, key, vals0[0]) == True - assert dber.addIoVal(db, key, vals0[1]) == True + assert dber.addIoDupVal(db, key, vals0[0]) == True + assert dber.addIoDupVal(db, key, vals0[1]) == True vals1 = [b"mary", b"peter", b"john", b"paul"] sn += 1 key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals1) == True + assert dber.putIoDupVals(db, key, vals1) == True vals2 = [b"dog", b"cat", b"bird"] sn += 2 # gap key = snKey(pre, sn) - assert dber.putIoVals(db, key, vals2) == True + assert dber.putIoDupVals(db, key, vals2) == True - vals = [bytes(val) for val in dber.getIoValsAnyPreIter(db, pre)] allvals = vals0 + vals1 + vals2 + vals = [bytes(val) for key, val in dber.getTopIoDupItemIter(db, pre)] + # dber.getTopIoDupItemIter() assert vals == allvals - # Setup Tests for getIoItemsNext and getIoItemsNextIter + + + # Setup Tests for TopIoDupItemsIter edb = dber.env.open_db(key=b'escrow.', dupsort=True) aKey = snKey(pre=b'A', sn=1) aVals = [b"z", b"m", b"x"] @@ -560,138 +629,387 @@ def test_lmdber(): dKey = snKey(pre=b'A', sn=7) dVals = [b"k", b"b"] - assert dber.putIoVals(edb, key=aKey, vals=aVals) - assert dber.putIoVals(edb, key=bKey, vals=bVals) - assert dber.putIoVals(edb, key=cKey, vals=cVals) - assert dber.putIoVals(edb, key=dKey, vals=dVals) - - # Test getIoItemsNext(self, db, key=b"") - # aVals - items = dber.getIoItemsNext(edb) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoItemsNext(edb, key=aKey, skip=False) # get aKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = dber.getIoItemsNext(edb, key=aKey) # get bKey in database - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = dber.getIoItemsNext(edb, key=b'', skip=False) # get first key in database - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - # bVals - items = dber.getIoItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - # cVals - items = dber.getIoItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - - # dVals - items = dber.getIoItemsNext(edb, key=ikey) - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - - # none - items = dber.getIoItemsNext(edb, key=ikey) - assert items == [] # empty + + # Test variousItemIter + assert dber.putIoDupVals(edb, key=aKey, vals=aVals) + assert dber.putIoDupVals(edb, key=bKey, vals=bVals) + assert dber.putIoDupVals(edb, key=cKey, vals=cVals) + assert dber.putIoDupVals(edb, key=dKey, vals=dVals) + + # test getTopIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] + + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items - # Test getIoItemsNextIter(self, db, key=b"") - # get dups at first key in database - # aVals - items = [item for item in dber.getIoItemsNextIter(edb)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey, skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - - items = [item for item in dber.getIoItemsNextIter(edb, key=aKey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - - items = [item for item in dber.getIoItemsNextIter(edb, key=b'', skip=False)] - assert items # not empty - ikey = items[0][0] - assert ikey == aKey - vals = [val for key, val in items] - assert vals == aVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # bVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == bKey - vals = [val for key, val in items] - assert vals == bVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # cVals - items = [item for item in dber.getIoItemsNextIter(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == cKey - vals = [val for key, val in items] - assert vals == cVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # dVals - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] - assert items # not empty - ikey = items[0][0] - assert ikey == dKey - vals = [val for key, val in items] - assert vals == dVals - for key, val in items: - assert dber.delIoVal(edb, ikey, val) == True - - # none - items = [item for item in dber.getIoItemsNext(edb, key=ikey)] - assert items == [] # empty + + # test getIoDupItemIter + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb)] # default all + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x'), + (b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z'), + (b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n'), + (b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + + # dups at aKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=aKey)] + assert items == [(b'A.00000000000000000000000000000001', b'z'), + (b'A.00000000000000000000000000000001', b'm'), + (b'A.00000000000000000000000000000001', b'x')] + # dups at bKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=bKey)] + assert items == [(b'A.00000000000000000000000000000002', b'o'), + (b'A.00000000000000000000000000000002', b'r'), + (b'A.00000000000000000000000000000002', b'z')] + + # dups at cKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=cKey)] + assert items == [(b'A.00000000000000000000000000000004', b'h'), + (b'A.00000000000000000000000000000004', b'n')] + + # dups at dKey + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=dKey)] + assert items == [(b'A.00000000000000000000000000000007', b'k'), + (b'A.00000000000000000000000000000007', b'b')] + + # dups at missing key + items = [(ikey, bytes(ival)) for ikey, ival in dber.getTopIoDupItemIter(edb, top=b"B.")] assert not items + + # test OnIoDup methods + ldb = dber.env.open_db(key=b'log.', dupsort=True) + # first pre + sn = 0 + key = snKey(preA, sn) + valsA0 = [b"echo", b"bravo"] + itemsA0 = [ + (preA, sn, valsA0[0]), + (preA, sn, valsA0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsA0[0]) == True + assert dber.addIoDupVal(ldb, key, valsA0[1]) == True + + sn += 1 + key = snKey(preA, sn) + valsA1 = [b"sue", b"bob", b"val", b"zoe"] + itemsA1 = [ + (preA, sn, valsA1[0]), + (preA, sn, valsA1[1]), + (preA, sn, valsA1[2]), + (preA, sn, valsA1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsA1) == True + + sn += 1 + key = snKey(preA, sn) + valsA2 = [b"fish", b"bat", b"snail"] + itemsA2 = [ + (preA, sn, valsA2[0]), + (preA, sn, valsA2[1]), + (preA, sn, valsA2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsA2) == True + + # second pre + sn = 0 + key = snKey(preB, sn) + valsB0 = [b"gamma", b"beta"] + itemsB0 = [ + (preB, sn, valsB0[0]), + (preB, sn, valsB0[1]) + ] + assert dber.addIoDupVal(ldb, key, valsB0[0]) == True + assert dber.addIoDupVal(ldb, key, valsB0[1]) == True + + sn += 1 + key = snKey(preB, sn) + valsB1 = [b"mary", b"peter", b"john", b"paul"] + itemsB1 = [ + (preB, sn, valsB1[0]), + (preB, sn, valsB1[1]), + (preB, sn, valsB1[2]), + (preB, sn, valsB1[3]), + ] + assert dber.putIoDupVals(ldb, key, valsB1) == True + + + sn += 1 + key = snKey(preB, sn) + valsB2 = [b"dog", b"cat", b"bird"] + itemsB2 = [ + (preB, sn, valsB2[0]), + (preB, sn, valsB2[1]), + (preB, sn, valsB2[2]), + ] + assert dber.putIoDupVals(ldb, key, valsB2) == True + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preA, on=1)] + lastitems = [itemsA1[-1], itemsA2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1]] + assert vals == lastvals + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preA, on=1)] + lastvals = [valsA1[-1], valsA2[-1]] + assert vals == lastvals + + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb, preB)] + lastitems = [itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb, preB)] + lastvals = [valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + items = [(key, on, bytes(val)) for key, on, val in dber.getOnIoDupLastItemIter(ldb)] + lastitems = [itemsA0[-1], itemsA1[-1], itemsA2[-1], itemsB0[-1], itemsB1[-1], itemsB2[-1]] + assert items == lastitems + + vals = [bytes(val) for val in dber.getOnIoDupLastValIter(ldb)] + lastvals = [valsA0[-1], valsA1[-1], valsA2[-1], valsB0[-1], valsB1[-1], valsB2[-1]] + assert vals == lastvals + + # test back iter + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=3)] + assert items ==[(preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma')] + + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=3)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preB, on=1)] + assert items ==[ + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma') + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preB, on=1)] + assert vals ==[ + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=5)] + assert items ==[(preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=5)] + assert vals ==[ + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=preA, on=0)] + assert items ==[ + (preA, 0, b'bravo'), + (preA, 0, b'echo')] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb, key=preA, on=0)] + assert vals ==[ + b'bravo', + b'echo' + ] + + # all items from last to first + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb)] + assert items ==[ + (preB, 2, b'bird'), + (preB, 2, b'cat'), + (preB, 2, b'dog'), + (preB, 1, b'paul'), + (preB, 1, b'john'), + (preB, 1, b'peter'), + (preB, 1, b'mary'), + (preB, 0, b'beta'), + (preB, 0, b'gamma'), + (preA, 2, b'snail'), + (preA, 2, b'bat'), + (preA, 2, b'fish'), + (preA, 1, b'zoe'), + (preA, 1, b'val'), + (preA, 1, b'bob'), + (preA, 1, b'sue'), + (preA, 0, b'bravo'), + (preA, 0, b'echo'), + ] + + vals = [ bytes(val) for val in dber.getOnIoDupValBackIter(ldb)] + assert vals ==[ + b'bird', + b'cat', + b'dog', + b'paul', + b'john', + b'peter', + b'mary', + b'beta', + b'gamma', + b'snail', + b'bat', + b'fish', + b'zoe', + b'val', + b'bob', + b'sue', + b'bravo', + b'echo' + ] + + + # test OnIoDup methods + key = b'Z' + assert 0 == dber.appendOnIoDupVal(ldb, key, val=b'k') + assert 1 == dber.appendOnIoDupVal(ldb, key, val=b'l') + assert 2 == dber.appendOnIoDupVal(ldb, key, val=b'm') + assert 3 == dber.appendOnIoDupVal(ldb, key, val=b'n') + + assert dber.cntOnVals(ldb, key) == 4 + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key)] + assert vals == [b'k', b'l', b'm', b'n'] + + vals = [ bytes(val) for val in dber.getOnIoDupValIter(ldb, key=key, on=2)] + assert vals == [ b'm', b'n'] + + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Z', 0, b'k'), + (b'Z', 1, b'l'), + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key, on=2)] + assert items == [ + (b'Z', 2, b'm'), + (b'Z', 3, b'n')] + + # test back iter + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=3)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=4)] + assert items ==[(b'Z', 3, b'n'), + (b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemBackIter(ldb, key=key, on=2)] + assert items == [(b'Z', 2, b'm'), + (b'Z', 1, b'l'), + (b'Z', 0, b'k')] + + key = b'Y' + assert dber.addOnIoDupVal(ldb, key, on=0, val=b'r') + assert dber.addOnIoDupVal(ldb, key, on=0, val=b's') + assert dber.addOnIoDupVal(ldb, key, on=1, val=b't') + assert dber.addOnIoDupVal(ldb, key, on=1, val=b'u') + + assert dber.cntOnVals(ldb, key) == 4 + + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Y', 0, b'r'), + (b'Y', 0, b's'), + (b'Y', 1, b't'), + (b'Y', 1, b'u')] + + assert dber.delOnIoDupVal(ldb, key, on=0, val=b's') + assert dber.delOnIoDupVals(ldb, key, on=1) + items = [ (key, on, bytes(val)) for key, on, val in dber.getOnIoDupItemIter(ldb, key=key)] + assert items == [(b'Y', 0, b'r')] + # test IoSetVals insertion order set of vals methods. key0 = b'ABC.ZYX' key1 = b'DEF.WVU' @@ -773,23 +1091,6 @@ def test_lmdber(): assert dber.getIoSetVals(db, key1) == vals1 assert dber.getIoSetVals(db, key2) == vals2 - assert dber.appendIoSetVal(db, key1, val=b"k") == 4 - assert dber.getIoSetVals(db, key1) == [b"w", b"n", b"y", b"d", b"k"] - - assert dber.getIoSetItems(db, key0) == [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')] - - assert ([(bytes(iokey), bytes(val)) for iokey, val in dber.getIoSetItemsIter(db, key0)] == - [(b'ABC.ZYX.00000000000000000000000000000000', b'z'), - (b'ABC.ZYX.00000000000000000000000000000001', b'm'), - (b'ABC.ZYX.00000000000000000000000000000002', b'x'), - (b'ABC.ZYX.00000000000000000000000000000003', b'a')]) - - for iokey, val in dber.getIoSetItemsIter(db, key0): - assert dber.delIoSetIokey(db, iokey) - assert dber.getIoSetVals(db, key0) == [] vals3 = [b"q", b"e"] assert dber.setIoSetVals(db, key2, vals3) @@ -811,17 +1112,12 @@ def test_lmdber(): dber.putIoSetVals(db, empty_key, [some_value]) dber.addIoSetVal(db, empty_key, some_value) dber.setIoSetVals(db, empty_key, [some_value]) - dber.appendIoSetVal(db, empty_key, some_value) dber.getIoSetVals(db, empty_key) [_ for _ in dber.getIoSetValsIter(db, empty_key)] dber.getIoSetValLast(db, empty_key) dber.cntIoSetVals(db, empty_key) dber.delIoSetVals(db, empty_key) dber.delIoSetVal(db, empty_key, some_value) - dber.getIoSetItems(db, empty_key) - dber.getIoSetItemsIter(db, empty_key) - with pytest.raises(KeyError): - dber.delIoSetIokey(db, empty_key) with pytest.raises(KeyError): dber.putVals(db, empty_key, [some_value]) with pytest.raises(KeyError): @@ -837,21 +1133,21 @@ def test_lmdber(): with pytest.raises(KeyError): dber.delVals(db, empty_key) with pytest.raises(KeyError): - dber.putIoVals(db, empty_key, [some_value]) + dber.putIoDupVals(db, empty_key, [some_value]) with pytest.raises(KeyError): - dber.addIoVal(db, empty_key, some_value) + dber.addIoDupVal(db, empty_key, some_value) with pytest.raises(KeyError): - dber.getIoVals(db, empty_key) + dber.getIoDupVals(db, empty_key) with pytest.raises(KeyError): - [_ for _ in dber.getIoValsIter(db, empty_key)] + [_ for _ in dber.getIoDupValsIter(db, empty_key)] with pytest.raises(KeyError): - dber.getIoValLast(db, empty_key) + dber.getIoDupValLast(db, empty_key) with pytest.raises(KeyError): - dber.cntIoVals(db, empty_key) + dber.cntIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVals(db, empty_key) + dber.delIoDupVals(db, empty_key) with pytest.raises(KeyError): - dber.delIoVal(db, empty_key, some_value) + dber.delIoDupVal(db, empty_key, some_value) assert not os.path.exists(dber.path) @@ -860,4 +1156,6 @@ def test_lmdber(): if __name__ == "__main__": test_key_funcs() + test_suffix() test_lmdber() + test_opendatabaser() diff --git a/tests/db/test_escrowing.py b/tests/db/test_escrowing.py index 9ca309126..6a7e5af24 100644 --- a/tests/db/test_escrowing.py +++ b/tests/db/test_escrowing.py @@ -4,8 +4,12 @@ """ from keri import kering -from keri.app import habbing + +from keri import core from keri.core import coring, eventing, serdering + +from keri.app import habbing + from keri.core.eventing import SealEvent from keri.db import escrowing, dbing, subing from keri.help import helping @@ -27,7 +31,7 @@ def test_broker(): def test_broker_nontrans(): raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - salter = coring.Salter(raw=raw) + salter = core.Salter(raw=raw) salt = salter.qb64 assert salt == '0AAFqo8tU5rp-lWcApybCEh1' @@ -62,7 +66,7 @@ def test_broker_nontrans(): rrsr = viring.RegStateRecord._fromdict(ked["a"]) # reply RegStateRecord #tserder = serdering.SerderKERI(sad=ked["a"]) - saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Serials.json, label=coring.Saids.d) + saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Kinds.json, label=coring.Saids.d) dater = coring.Dater(dts=dts) cigars = wesHab.sign(ser=serder.raw, @@ -128,7 +132,7 @@ def test_broker_trans(): serder = serdering.SerderKERI(sad=ked) rrsr = viring.RegStateRecord._fromdict(ked["a"]) # reply RegStateRecord #tserder = serdering.SerderKERI(sad=ked["a"]) - saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Serials.json, label=coring.Saids.d) + saider, _ = coring.Saider.saidify(sad=ked, kind=coring.Kinds.json, label=coring.Saids.d) dater = coring.Dater(dts=dts) sigers = bobHab.sign(ser=serder.raw, diff --git a/tests/db/test_koming.py b/tests/db/test_koming.py index dfafa5327..ecc58ff9d 100644 --- a/tests/db/test_koming.py +++ b/tests/db/test_koming.py @@ -10,7 +10,7 @@ import pytest -from keri.core.coring import Serials +from keri.core.coring import Kinds from keri.db import dbing, koming from keri.help import helping @@ -299,16 +299,16 @@ class Record: with dbing.openLMDB() as db: k = koming.Komer(db=db, schema=Record, subkey='records.') - srl = k._serializer(Serials.mgpk) + srl = k._serializer(Kinds.mgpk) expected = b'\x86\xa5first\xa3Jim\xa4last\xa5Black\xa6street\xaf100 Main Street\xa4city\xa8Riverton\xa5state\xa2UT\xa3zip\xce\x00\x01HZ' assert srl(jim) == expected - srl = k._serializer(Serials.cbor) + srl = k._serializer(Kinds.cbor) expected = b'\xa6efirstcJimdlasteBlackfstreeto100 Main StreetdcityhRivertonestatebUTczip\x1a\x00\x01HZ' assert srl(jim) == expected - srl = k._serializer(Serials.json) + srl = k._serializer(Kinds.json) expected = b'{"first":"Jim","last":"Black","street":"100 Main Street","city":"Riverton","state":"UT","zip":84058}' assert srl(jim) == expected @@ -389,7 +389,7 @@ class Record: with dbing.openLMDB() as db: k = koming.Komer(db=db, schema=Record, subkey='records.') - desrl = k._deserializer(Serials.mgpk) + desrl = k._deserializer(Kinds.mgpk) actual = helping.datify(Record, desrl(msgp)) assert actual.first == "Jim" assert actual.last == "Black" @@ -398,7 +398,7 @@ class Record: assert actual.state == "UT" assert actual.zip == 84058 - desrl = k._deserializer(Serials.json) + desrl = k._deserializer(Kinds.json) actual = helping.datify(Record, desrl(json)) assert actual.first == "Jim" assert actual.last == "Black" @@ -407,7 +407,7 @@ class Record: assert actual.state == "UT" assert actual.zip == 84058 - desrl = k._deserializer(Serials.cbor) + desrl = k._deserializer(Kinds.cbor) actual = helping.datify(Record, desrl(cbor)) assert actual.first == "Jim" assert actual.last == "Black" @@ -758,7 +758,7 @@ def __iter__(self): loc = locDB.get(keys=(end.eid, scheme)) assert loc == wit3loc - # test IoItem methods + ## test IoItem methods iokeys0 = [f'{cid0}.witness.00000000000000000000000000000000'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000001'.encode("utf-8"), f'{cid0}.witness.00000000000000000000000000000002'.encode("utf-8")] @@ -773,20 +773,13 @@ def __iter__(self): 'witness', '00000000000000000000000000000002')] - + # test getItemIter i = 0 - for iokeys, end in endDB.getIoSetItem(keys=keys0): + for keys, end in endDB.getItemIter(keys=keys0): assert end == ends[i] - assert iokeys == iokeys0[i] + assert keys == keys0 i += 1 - i = 0 - for iokeys, end in endDB.getIoSetItemIter(keys=keys0): - assert end == ends[i] - assert iokeys == iokeys0[i] - i += 1 - - # test getAllItemIter ends = ends + [wit3end] i = 0 for keys, end in endDB.getItemIter(): @@ -829,27 +822,29 @@ def __iter__(self): '00000000000000000000000000000000')] i = 0 - for iokeys, end in endDB.getIoItemIter(keys=(cid0, "")): + for iokeys, end in endDB.getFullItemIter(keys=(cid0, "")): assert end == ends[i] assert iokeys == iokeysall[i] i += 1 - i = 0 - for iokeys, end in endDB.getIoItemIter(): - assert end == ends[i] - assert iokeys == iokeysall[i] - i += 1 - assert endDB.remIokey(iokeys) - - assert endDB.cnt(keys0) == 0 - assert endDB.cnt(keys1) == 0 + #for iokeys, val in endDB.getFullItemIter(): + #assert endDB.remIokey(iokeys=iokeys) + #assert endDB.cnt(keys=keys0) == 0 + #assert endDB.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) assert not db.opened if __name__ == "__main__": - test_dup_komer() + test_kom_happy_path() test_kom_get_item_iter() + test_put_invalid_dataclass() + test_get_invalid_dataclass() + test_not_found_entity() + test_serialization() + test_custom_serialization() + test_deserialization() + test_dup_komer() test_ioset_komer() diff --git a/tests/db/test_subing.py b/tests/db/test_subing.py index c777dc278..58c72ccdc 100644 --- a/tests/db/test_subing.py +++ b/tests/db/test_subing.py @@ -9,10 +9,14 @@ import pysodium -from keri.core import coring, eventing, serdering +from keri import help + +from keri import core +from keri.core import coring, eventing, serdering, indexing, scheming + from keri.db import dbing, subing from keri.app import keeping -from keri.help import helping + @@ -27,56 +31,85 @@ def test_suber(): assert db.name == "test" assert db.opened - sdb = subing.Suber(db=db, subkey='bags.') - assert isinstance(sdb, subing.Suber) - assert not sdb.sdb.flags()["dupsort"] + suber = subing.Suber(db=db, subkey='bags.') + assert isinstance(suber, subing.Suber) + assert not suber.sdb.flags()["dupsort"] sue = "Hello sailer!" keys = ("test_key", "0001") - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue - sdb.rem(keys) - actual = sdb.get(keys=keys) + suber.rem(keys) + actual = suber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=sue) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) assert actual == sue kip = "Hey gorgeous!" - result = sdb.put(keys=keys, val=kip) + result = suber.put(keys=keys, val=kip) assert not result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == sue - result = sdb.pin(keys=keys, val=kip) + result = suber.pin(keys=keys, val=kip) assert result - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual == kip + suber.rem(keys) + actual = suber.get(keys=keys) + assert actual is None + + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) + assert actual == sue + + # test with keys as tuple of bytes + keys = (b"test_key", b"0001") + suber.rem(keys) + actual = suber.get(keys=keys) + assert actual is None + + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) + assert actual == sue + + # test with keys as mixed tuple of bytes + keys = (b"test_key", "0001") + suber.rem(keys) + actual = suber.get(keys=keys) + assert actual is None + + suber.put(keys=keys, val=sue) + actual = suber.get(keys=keys) + assert actual == sue + + # test with keys as string not tuple keys = "keystr" bob = "Shove off!" - sdb.put(keys=keys, val=bob) - actual = sdb.get(keys=keys) + suber.put(keys=keys, val=bob) + actual = suber.get(keys=keys) assert actual == bob - sdb.rem(keys) + suber.rem(keys) - actual = sdb.get(keys=keys) + actual = suber.get(keys=keys) assert actual is None liz = "May life is insane." keys = ("test_key", "0002") - sdb.put(keys=keys, val=liz) - actual = sdb.get(("not_found", "0002")) + suber.put(keys=keys, val=liz) + actual = suber.get(("not_found", "0002")) assert actual is None w = "Blue dog" @@ -84,26 +117,26 @@ def test_suber(): y = "Red apple" z = "White snow" - sdb = subing.Suber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.Suber) + suber = subing.Suber(db=db, subkey='pugs.') + assert isinstance(suber, subing.Suber) - sdb.put(keys=("a","1"), val=w) - sdb.put(keys=("a","2"), val=x) - sdb.put(keys=("a","3"), val=y) - sdb.put(keys=("a","4"), val=z) + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] - sdb.put(keys=("b","1"), val=w) - sdb.put(keys=("b","2"), val=x) - sdb.put(keys=("bc","3"), val=y) - sdb.put(keys=("ac","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + suber.put(keys=("bc","3"), val=y) + suber.put(keys=("ac","4"), val=z) - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -113,20 +146,36 @@ def test_suber(): (('b', '2'), 'Green tree'), (('bc', '3'), 'Red apple')] + + # test with top keys for partial tree topkeys = ("b","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), w), (('b', '2'), x)] topkeys = ("a","") # last element empty to force trailing separator - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in suber.getItemIter(keys=topkeys)] assert items == [(('a', '1'), w), (('a', '2'), x), (('a', '3'), y), (('a', '4'), z)] - assert sdb.trim(keys=("b", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + # test with top parameter + keys = ("b", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] + assert items == [(('b', '1'), w), + (('b', '2'), x)] + + keys = ("a", ) # last element empty to force trailing separator + items = [(keys, val) for keys, val in suber.getItemIter(keys=keys, topive=True)] + assert items == [(('a', '1'), w), + (('a', '2'), x), + (('a', '3'), y), + (('a', '4'), z)] + + # Test trim + assert suber.trim(keys=("b", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('a', '1'), 'Blue dog'), (('a', '2'), 'Green tree'), (('a', '3'), 'Red apple'), @@ -134,20 +183,279 @@ def test_suber(): (('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim(keys=("a", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert suber.trim(keys=("a", "")) + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [(('ac', '4'), 'White snow'), (('bc', '3'), 'Red apple')] - assert sdb.trim() - items = [(keys, val) for keys, val in sdb.getItemIter()] + # Test trim with top parameters + suber.put(keys=("a","1"), val=w) + suber.put(keys=("a","2"), val=x) + suber.put(keys=("a","3"), val=y) + suber.put(keys=("a","4"), val=z) + suber.put(keys=("b","1"), val=w) + suber.put(keys=("b","2"), val=x) + + assert suber.trim(keys=("b",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] + assert items == [(('a', '1'), 'Blue dog'), + (('a', '2'), 'Green tree'), + (('a', '3'), 'Red apple'), + (('a', '4'), 'White snow'), + (('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + + assert suber.trim(keys=("a",), topive=True) + items = [(keys, val) for keys, val in suber.getItemIter()] + assert items == [(('ac', '4'), 'White snow'), + (('bc', '3'), 'Red apple')] + + assert suber.trim() + items = [(keys, val) for keys, val in suber.getItemIter()] assert items == [] - assert not sdb.trim() + assert not suber.trim() assert not os.path.exists(db.path) assert not db.opened +def test_on_suber(): + """ + Test OnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + # test getOnIter + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + + + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 2 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 2, 'Red apple')] + + assert onsuber.putOn(keys='d', on=0, val='moon') + assert onsuber.getOn(keys='d', on=0) == 'moon' + assert not onsuber.putOn(keys='d', on=0, val='moon') + assert onsuber.pinOn(keys='d', on=0, val='sun') + assert onsuber.getOn(keys='d', on=0) == 'sun' + assert onsuber.remOn(keys='d', on=0) + + + + assert not os.path.exists(db.path) + assert not db.opened + + +def test_B64_suber(): + """ + Test B64Suber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + # Test Single klas + buber = subing.B64Suber(db=db, subkey='bags.') # default klas is [Matter] + assert isinstance(buber, subing.B64Suber) + assert not buber.sdb.flags()["dupsort"] + + vals0 = ("alpha", "beta") + vals1 = ("gamma", ) + + keys0 = ("cat", "dog") + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + assert buber.put(keys=keys0, val=vals0) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + + assert not buber.put(keys=keys0, val=vals1) + + assert buber.pin(keys=keys0, val=vals1) + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + assert not buber.get(keys=keys0) + + # test with vals as non Iterable or put but Iterable on get + assert buber.put(keys=keys0, val="gamma") + actuals = buber.get(keys=keys0) + assert actuals == vals1 + + assert buber.rem(keys0) + + # test val as already joined string + with pytest.raises(ValueError): + assert buber.put(keys=keys0, val="alpha.beta") + + # test bytes for val + assert buber.put(keys=keys0, val=(b"alpha", b"beta")) + actuals = buber.get(keys=keys0) + assert actuals == vals0 + assert buber.rem(keys0) + + # test with keys as string not tuple + keys1 = "{}.{}".format("bird", "fish") + + assert buber.put(keys=keys1, val=vals1) + actuals = buber.get(keys=keys1) + assert actuals == vals1 + + assert buber.rem(keys1) + assert not buber.get(keys=keys1) + + # test missing entry at keys + badkey = "badkey" + assert not buber.get(badkey) + + # test iteritems + assert buber.put(keys0, vals0) + assert buber.put(keys1, vals1) + + items = [ items for items in buber.getItemIter()] + assert items == [(('bird', 'fish'), ('gamma',)), (('cat', 'dog'), ('alpha', 'beta'))] + + buber.put(keys=("b","1"), val=vals0) + buber.put(keys=("b","2"), val=vals1) + buber.put(keys=("c","1"), val=vals0) + buber.put(keys=("c","2"), val=vals1) + + topkeys = ("b","") # last element empty to force trailing separator + items = [items for items in buber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), ('alpha', 'beta')), + (('b', '2'), ('gamma',))] + + assert not os.path.exists(db.path) + assert not db.opened + """Done Test""" + + def test_dup_suber(): """ Test DubSuber LMDBer sub database class @@ -158,54 +466,54 @@ def test_dup_suber(): assert db.name == "test" assert db.opened - sdb = subing.DupSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.DupSuber) - assert sdb.sdb.flags()["dupsort"] + dupber = subing.DupSuber(db=db, subkey='bags.') + assert isinstance(dupber, subing.DupSuber) + assert dupber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - sdb.put(keys=keys0, vals=[sue, sal]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sue, sal]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - assert sdb.cnt(keys0) == 2 + assert dupber.cnt(keys0) == 2 - sdb.rem(keys0) - actual = sdb.get(keys=keys0) + dupber.rem(keys0) + actual = dupber.get(keys=keys0) assert not actual assert actual == [] - assert sdb.cnt(keys0) == 0 + assert dupber.cnt(keys0) == 0 - sdb.put(keys=keys0, vals=[sal, sue]) - actual = sdb.get(keys=keys0) + dupber.put(keys=keys0, vals=[sal, sue]) + actual = dupber.get(keys=keys0) assert actual == [sue, sal] # lexicographic order - actual = sdb.getLast(keys=keys0) + actual = dupber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = dupber.add(keys=keys0, val=sam) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [sam, sue, sal] # lexicographic order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = dupber.pin(keys=keys0, vals=[zoe, zia]) assert result - actual = sdb.get(keys=keys0) + actual = dupber.get(keys=keys0) assert actual == [zia, zoe] # lexi order - sdb.put(keys=keys1, vals=[sal, sue, sam]) - actual = sdb.get(keys=keys1) + dupber.put(keys=keys1, vals=[sal, sue, sam]) + actual = dupber.get(keys=keys1) assert actual == [sam, sue, sal] # lexicographic order - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(dupber.getIter(keys=keys1)): assert val == actual[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in dupber.getItemIter()] assert items == [(('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0001'), 'See ya later.'), (('test_key', '0002'), 'A real charmer!'), @@ -213,9 +521,9 @@ def test_dup_suber(): (('test_key', '0002'), 'Not my type.')] - assert sdb.put(keys=("test", "blue"), vals=[sal, sue, sam]) + assert dupber.put(keys=("test", "blue"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in dupber.getItemIter(keys=topkeys)] assert items == [(('test', 'blue'), 'A real charmer!'), (('test', 'blue'), 'Hello sailer!'), (('test', 'blue'), 'Not my type.')] @@ -223,32 +531,521 @@ def test_dup_suber(): # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] - assert sdb.cnt(keys2) == 1 - sdb.rem(keys2) - actual = sdb.get(keys=keys2) + assert dupber.cnt(keys2) == 1 + dupber.rem(keys2) + actual = dupber.get(keys=keys2) assert actual == [] - assert sdb.cnt(keys2) == 0 + assert dupber.cnt(keys2) == 0 - sdb.put(keys=keys2, vals=[bob]) - actual = sdb.get(keys=keys2) + dupber.put(keys=keys2, vals=[bob]) + actual = dupber.get(keys=keys2) assert actual == [bob] bil = "Go away." - sdb.pin(keys=keys2, vals=[bil]) - actual = sdb.get(keys=keys2) + dupber.pin(keys=keys2, vals=[bil]) + actual = dupber.get(keys=keys2) assert actual == [bil] - sdb.add(keys=keys2, val=bob) - actual = sdb.get(keys=keys2) + dupber.add(keys=keys2, val=bob) + actual = dupber.get(keys=keys2) assert actual == [bil, bob] + + # test cntpre + vals = ["hi", "me", "my"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + vals = ["bye", "guy", "gal"] + for i, val in enumerate(vals): + assert dupber.put(keys=dbing.onKey("bob", i), vals=val) + + items = [(keys, val) for keys, val in dupber.getItemIter(keys="bob")] + assert items == [ + (('bob', '00000000000000000000000000000000'), 'bye'), + (('bob', '00000000000000000000000000000000'), 'hi'), + (('bob', '00000000000000000000000000000001'), 'guy'), + (('bob', '00000000000000000000000000000001'), 'me'), + (('bob', '00000000000000000000000000000002'), 'gal'), + (('bob', '00000000000000000000000000000002'), 'my') + ] + + assert dupber.cnt(keys=dbing.onKey("bob", 1)) == 2 + + vals = [val for val in dupber.getIter(keys=dbing.onKey("bob", 2))] + assert vals == ["gal", "my"] + + + assert not os.path.exists(db.path) + assert not db.opened + + +def test_iodup_suber(): + """ + Test IoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + ioduber = subing.IoDupSuber(db=db, subkey='bags.') + assert isinstance(ioduber, subing.IoDupSuber) + assert ioduber.sdb.flags()["dupsort"] + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") + + sue = "Hello sailer!" + sal = "Not my type." + + keys0 = ("test_key", "0001") + keys1 = ("test_key", "0002") + + assert ioduber.put(keys=keys0, vals=[sal, sue]) + actuals = ioduber.get(keys=keys0) + assert actuals == [sal, sue] # insertion order not lexicographic + assert ioduber.cnt(keys0) == 2 + actual = ioduber.getLast(keys=keys0) + assert actual == sue + + assert ioduber.rem(keys0) + actuals = ioduber.get(keys=keys0) + assert not actuals + assert actuals == [] + assert ioduber.cnt(keys0) == 0 + + assert ioduber.put(keys=keys0, vals=[sue, sal]) + actuals = ioduber.get(keys=keys0) + assert actuals == [sue, sal] # insertion order + actual = ioduber.getLast(keys=keys0) + assert actual == sal + + sam = "A real charmer!" + result = ioduber.add(keys=keys0, val=sam) + assert result + actuals = ioduber.get(keys=keys0) + assert actuals == [sue, sal, sam] # insertion order + + zoe = "See ya later." + zia = "Hey gorgeous!" + + result = ioduber.pin(keys=keys0, vals=[zoe, zia]) + assert result + actuals = ioduber.get(keys=keys0) + assert actuals == [zoe, zia] # insertion order + + assert ioduber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = ioduber.get(keys=keys1) + assert actuals == [sal, sue, sam] + + for i, val in enumerate(ioduber.getIter(keys=keys1)): + assert val == actuals[i] + + items = [(keys, val) for keys, val in ioduber.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + items = list(ioduber.getFullItemIter()) + assert items == [(('test_key', '0001'), '00000000000000000000000000000000.See ya later.'), + (('test_key', '0001'), '00000000000000000000000000000001.Hey gorgeous!'), + (('test_key', '0002'), '00000000000000000000000000000000.Not my type.'), + (('test_key', '0002'), '00000000000000000000000000000001.Hello sailer!'), + (('test_key', '0002'), '00000000000000000000000000000002.A real charmer!')] + + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] + + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] + + + + + # Test with top keys + assert ioduber.put(keys=("test", "pop"), vals=[sal, sue, sam]) + topkeys = ("test", "") + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=topkeys)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in ioduber.getItemIter(keys=keys, topive=True)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # IoItems + items = list(ioduber.getFullItemIter(keys=topkeys)) + assert items == [(('test', 'pop'), '00000000000000000000000000000000.Not my type.'), + (('test', 'pop'), '00000000000000000000000000000001.Hello sailer!'), + (('test', 'pop'), '00000000000000000000000000000002.A real charmer!')] + + # test remove with a specific val + assert ioduber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in ioduber.getItemIter()] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!'), + (('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + assert ioduber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in ioduber.getItemIter()] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!'), + (('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'A real charmer!')] + + assert ioduber.cnt(keys=keys0) == 2 + assert ioduber.cnt(keys=keys1) == 2 + + # test with keys as string not tuple + keys2 = "keystr" + bob = "Shove off!" + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) + assert actuals == [bob] + assert ioduber.cnt(keys2) == 1 + assert ioduber.rem(keys2) + actuals = ioduber.get(keys=keys2) + assert actuals == [] + assert ioduber.cnt(keys2) == 0 + + assert ioduber.put(keys=keys2, vals=[bob]) + actuals = ioduber.get(keys=keys2) + assert actuals == [bob] + + bil = "Go away." + assert ioduber.pin(keys=keys2, vals=[bil]) + actuals = ioduber.get(keys=keys2) + assert actuals == [bil] + + assert ioduber.add(keys=keys2, val=bob) + actuals = ioduber.get(keys=keys2) + assert actuals == [bil, bob] + + # Test trim + assert ioduber.trim() # default trims whole database + assert ioduber.put(keys=keys1, vals=[bob, bil]) + assert ioduber.get(keys=keys1) == [bob, bil] + + assert not os.path.exists(db.path) assert not db.opened +def test_on_iodup_suber(): + """ + Test OnIoDupSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.OnIoDupSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.OnIoDupSuber) + assert onsuber.sdb.flags()["dupsort"] + + w = "Blue dog" + x = "Green tree" + y = "Red apple" + z = "White snow" + + # test addOn remOn + assert onsuber.addOn(keys="z", on=0, val=w) + assert onsuber.getOn(keys="z", on=0) == [w] + assert onsuber.addOn(keys="z", on=0, val=x) + assert onsuber.getOn(keys="z", on=0) == [w, x] + assert onsuber.addOn(keys="z", on=1, val=y) + assert onsuber.getOn(keys="z", on=1) == [y] + assert onsuber.addOn(keys="z", on=1, val=z) + assert onsuber.getOn(keys="z", on=1) == [y, z] + + assert onsuber.cntOn(keys=("z",)) == 4 + + items = [item for item in onsuber.getOnItemIter(keys='z')] + assert items == [(('z',), 0, 'Blue dog'), + (('z',), 0, 'Green tree'), + (('z',), 1, 'Red apple'), + (('z',), 1, 'White snow')] + + assert onsuber.remOn(keys='z', on=0, val=w) + assert onsuber.remOn(keys='z', on=1) + items = [item for item in onsuber.getOnItemIter(keys='z')] + assert items == [(('z',), 0, 'Green tree')] + assert onsuber.remOn(keys='z', on=0, val=x) + + assert onsuber.cntOn(keys=("z",)) == 0 + + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow')] + + + # test getOnIter + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow'] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == ['Red apple', + 'White snow'] + + # test getOnItemIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, 'Red apple'), + (('a',), 3, 'White snow')] + + # test add duplicates + assert onsuber.add(keys=dbing.onKey("b", 0), val=w) + assert onsuber.add(keys=dbing.onKey("b", 1), val=x) + assert onsuber.add(keys=dbing.onKey("bc", 0), val=y) + assert onsuber.add(keys=dbing.onKey("ac", 0), val=z) + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('ac', '00000000000000000000000000000000'), 'White snow'), + (('b', '00000000000000000000000000000000'), 'Blue dog'), + (('b', '00000000000000000000000000000001'), 'Green tree'), + (('bc', '00000000000000000000000000000000'), 'Red apple')] + + + # test getOnItemIter getOnIter + items = [item for item in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + vals = [val for val in onsuber.getOnIter(keys='b')] + assert vals == ['Blue dog', 'Green tree'] + + vals = [val for val in onsuber.getOnIter(keys=('b', ))] + assert vals == ['Blue dog', 'Green tree'] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree')] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + vals = [val for val in onsuber.getOnIter(keys=('b', ""))] + assert vals == [] + + items = [item for item in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + vals = [val for val in onsuber.getOnIter(keys='')] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + items = [item for item in onsuber.getOnItemIter()] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 1, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 3, 'White snow'), + (('ac',), 0, 'White snow'), + (('b',), 0, 'Blue dog'), + (('b',), 1, 'Green tree'), + (('bc',), 0, 'Red apple')] + + vals = [val for val in onsuber.getOnIter()] + assert vals == ['Blue dog', + 'Green tree', + 'Red apple', + 'White snow', + 'White snow', + 'Blue dog', + 'Green tree', + 'Red apple'] + + # test with duplicates + assert onsuber.add(keys=dbing.onKey("a", 0), val=z) + assert onsuber.add(keys=dbing.onKey("a", 1), val=y) + assert onsuber.add(keys=dbing.onKey("a", 2), val=x) + assert onsuber.add(keys=dbing.onKey("a", 3), val=w) + + assert onsuber.cntOn(keys=("a",)) == 8 + assert onsuber.cntOn(keys=("a",), on=2) == 4 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val) for keys, val in onsuber.getItemIter(keys=("a", ""))] + assert items == [(('a', '00000000000000000000000000000000'), 'Blue dog'), + (('a', '00000000000000000000000000000000'), 'White snow'), + (('a', '00000000000000000000000000000001'), 'Green tree'), + (('a', '00000000000000000000000000000001'), 'Red apple'), + (('a', '00000000000000000000000000000002'), 'Red apple'), + (('a', '00000000000000000000000000000002'), 'Green tree'), + (('a', '00000000000000000000000000000003'), 'White snow'), + (('a', '00000000000000000000000000000003'), 'Blue dog')] + + # test getOnItemIter getOnIter getOnItemBackIter getOnBackIter + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 1, 'Green tree'), + (('a',), 1, 'Red apple'), + (('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=4)] + assert items == [(('a',), 3, 'Blue dog'), + (('a',), 3, 'White snow'), + (('a',), 2, 'Green tree'), + (('a',), 2, 'Red apple'), + (('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=4)] + assert vals == ['Blue dog', + 'White snow', + 'Green tree', + 'Red apple', + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] + + items = [item for item in onsuber.getOnItemIter(keys='a', on=2)] + assert items ==[(('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 3, 'White snow'), + (('a',), 3, 'Blue dog')] + + vals = [val for val in onsuber.getOnIter(keys='a', on=2)] + assert vals == [ + 'Red apple', + 'Green tree', + 'White snow', + 'Blue dog', + ] + + items = [item for item in onsuber.getOnItemBackIter(keys='a', on=1)] + assert items ==[(('a',), 1, 'Red apple'), + (('a',), 1, 'Green tree'), + (('a',), 0, 'White snow'), + (('a',), 0, 'Blue dog')] + + + vals = [val for val in onsuber.getOnBackIter(keys='a', on=1)] + assert vals == ['Red apple', + 'Green tree', + 'White snow', + 'Blue dog'] + + # test append with duplicates + assert 4 == onsuber.appendOn(keys=("a",), val=x) + assert onsuber.cntOn(keys=("a",)) == 9 + + # test remove + assert onsuber.remOn(keys='a', on=1) + assert not onsuber.remOn(keys='a', on=1) + assert onsuber.remOn(keys='a', on=3) + assert not onsuber.remOn(keys='a', on=3) + + assert onsuber.cntOn(keys=("a",)) == 5 + + items = [item for item in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, 'Blue dog'), + (('a',), 0, 'White snow'), + (('a',), 2, 'Red apple'), + (('a',), 2, 'Green tree'), + (('a',), 4, 'Green tree')] + + vals = [val for val in onsuber.getOnIter(keys='a')] + assert vals == [ + 'Blue dog', + 'White snow', + 'Red apple', + 'Green tree', + 'Green tree', + ] + + + + + assert not os.path.exists(db.path) + assert not db.opened + + + def test_ioset_suber(): """ Test IoSetSuber LMDBer sub database class @@ -259,9 +1056,9 @@ def test_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.IoSetSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.IoSetSuber) - assert not sdb.sdb.flags()["dupsort"] + iosuber = subing.IoSetSuber(db=db, subkey='bags.') + assert isinstance(iosuber, subing.IoSetSuber) + assert not iosuber.sdb.flags()["dupsort"] sue = "Hello sailer!" sal = "Not my type." @@ -269,85 +1066,95 @@ def test_ioset_suber(): keys0 = ("test_key", "0001") keys1 = ("test_key", "0002") - assert sdb.put(keys=keys0, vals=[sal, sue]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sal, sue]) + actuals = iosuber.get(keys=keys0) assert actuals == [sal, sue] # insertion order not lexicographic - assert sdb.cnt(keys0) == 2 - actual = sdb.getLast(keys=keys0) + assert iosuber.cnt(keys0) == 2 + actual = iosuber.getLast(keys=keys0) assert actual == sue - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert iosuber.rem(keys0) + actuals = iosuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert iosuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[sue, sal]) - actuals = sdb.get(keys=keys0) + assert iosuber.put(keys=keys0, vals=[sue, sal]) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal] # insertion order - actual = sdb.getLast(keys=keys0) + actual = iosuber.getLast(keys=keys0) assert actual == sal sam = "A real charmer!" - result = sdb.add(keys=keys0, val=sam) + result = iosuber.add(keys=keys0, val=sam) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [sue, sal, sam] # insertion order zoe = "See ya later." zia = "Hey gorgeous!" - result = sdb.pin(keys=keys0, vals=[zoe, zia]) + result = iosuber.pin(keys=keys0, vals=[zoe, zia]) assert result - actuals = sdb.get(keys=keys0) + actuals = iosuber.get(keys=keys0) assert actuals == [zoe, zia] # insertion order - assert sdb.put(keys=keys1, vals=[sal, sue, sam]) - actuals = sdb.get(keys=keys1) + assert iosuber.put(keys=keys1, vals=[sal, sue, sam]) + actuals = iosuber.get(keys=keys1) assert actuals == [sal, sue, sam] - for i, val in enumerate(sdb.getIter(keys=keys1)): + for i, val in enumerate(iosuber.getIter(keys=keys1)): assert val == actuals[i] - items = [(keys, val) for keys, val in sdb.getItemIter()] + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'Hello sailer!'), (('test_key', '0002'), 'A real charmer!')] - - items = list(sdb.getIoItemIter()) + items = list(iosuber.getFullItemIter()) assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!'), (('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = sdb.getIoSetItem(keys=keys1) - assert items == [(('test_key', '0002', '00000000000000000000000000000000'), 'Not my type.'), - (('test_key', '0002', '00000000000000000000000000000001'), 'Hello sailer!'), - (('test_key', '0002', '00000000000000000000000000000002'), 'A real charmer!')] - items = [(iokeys, val) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] - assert items == [(('test_key', '0001', '00000000000000000000000000000000'), 'See ya later.'), - (('test_key', '0001', '00000000000000000000000000000001'), 'Hey gorgeous!')] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys0)] + assert items == [(('test_key', '0001'), 'See ya later.'), + (('test_key', '0001'), 'Hey gorgeous!')] + + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys1)] + assert items == [(('test_key', '0002'), 'Not my type.'), + (('test_key', '0002'), 'Hello sailer!'), + (('test_key', '0002'), 'A real charmer!')] - assert sdb.put(keys=("test", "pop"), vals=[sal, sue, sam]) + + # Test with top keys + assert iosuber.put(keys=("test", "pop"), vals=[sal, sue, sam]) topkeys = ("test", "") - items = [(keys, val) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=topkeys)] + assert items == [(('test', 'pop'), 'Not my type.'), + (('test', 'pop'), 'Hello sailer!'), + (('test', 'pop'), 'A real charmer!')] + + # test with top parameter + keys = ("test", ) + items = [(keys, val) for keys, val in iosuber.getItemIter(keys=keys, topive=True)] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!')] - items = list(sdb.getIoItemIter(keys=topkeys)) + # IoItems + items = list(iosuber.getFullItemIter(keys=topkeys)) assert items == [(('test', 'pop', '00000000000000000000000000000000'), 'Not my type.'), (('test', 'pop', '00000000000000000000000000000001'), 'Hello sailer!'), (('test', 'pop', '00000000000000000000000000000002'), 'A real charmer!')] # test remove with a specific val - assert sdb.rem(keys=("test_key", "0002"), val=sue) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.rem(keys=("test_key", "0002"), val=sue) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test', 'pop'), 'Not my type.'), (('test', 'pop'), 'Hello sailer!'), (('test', 'pop'), 'A real charmer!'), @@ -356,45 +1163,45 @@ def test_ioset_suber(): (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - assert sdb.trim(keys=("test", "")) - items = [(keys, val) for keys, val in sdb.getItemIter()] + assert iosuber.trim(keys=("test", "")) + items = [(keys, val) for keys, val in iosuber.getItemIter()] assert items == [(('test_key', '0001'), 'See ya later.'), (('test_key', '0001'), 'Hey gorgeous!'), (('test_key', '0002'), 'Not my type.'), (('test_key', '0002'), 'A real charmer!')] - for iokeys, val in sdb.getIoItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 # test with keys as string not tuple keys2 = "keystr" bob = "Shove off!" - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] - assert sdb.cnt(keys2) == 1 - assert sdb.rem(keys2) - actuals = sdb.get(keys=keys2) + assert iosuber.cnt(keys2) == 1 + assert iosuber.rem(keys2) + actuals = iosuber.get(keys=keys2) assert actuals == [] - assert sdb.cnt(keys2) == 0 + assert iosuber.cnt(keys2) == 0 - assert sdb.put(keys=keys2, vals=[bob]) - actuals = sdb.get(keys=keys2) + assert iosuber.put(keys=keys2, vals=[bob]) + actuals = iosuber.get(keys=keys2) assert actuals == [bob] bil = "Go away." - assert sdb.pin(keys=keys2, vals=[bil]) - actuals = sdb.get(keys=keys2) + assert iosuber.pin(keys=keys2, vals=[bil]) + actuals = iosuber.get(keys=keys2) assert actuals == [bil] - assert sdb.add(keys=keys2, val=bob) - actuals = sdb.get(keys=keys2) + assert iosuber.add(keys=keys2, val=bob) + actuals = iosuber.get(keys=keys2) assert actuals == [bil, bob] + # Test trim and append + assert iosuber.trim() # default trims whole database + assert iosuber.put(keys=keys1, vals=[bob, bil]) + assert iosuber.get(keys=keys1) == [bob, bil] + assert not os.path.exists(db.path) assert not db.opened @@ -410,10 +1217,10 @@ def test_cesr_ioset_suber(): assert db.name == "test" assert db.opened - sdb = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) - assert isinstance(sdb, subing.CesrIoSetSuber) - assert issubclass(sdb.klas, coring.Saider) - assert not sdb.sdb.flags()["dupsort"] + cisuber = subing.CesrIoSetSuber(db=db, subkey='bags.', klas=coring.Saider) + assert isinstance(cisuber, subing.CesrIoSetSuber) + assert issubclass(cisuber.klas, coring.Saider) + assert not cisuber.sdb.flags()["dupsort"] seqner0 = coring.Seqner(sn=20) seq0 = seqner0.qb64 @@ -487,63 +1294,63 @@ def test_cesr_ioset_suber(): said2 = saider2.qb64 assert said2 == 'EJxOaEsBSObrcmrsnlfHOdVAowGhUBKoE2Ce3TZ4Mhgu' - assert sdb.put(keys=keys0, vals=[saider1, saider0]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider1, saider0]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said0] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said0 - assert sdb.rem(keys0) - actuals = sdb.get(keys=keys0) + assert cisuber.rem(keys0) + actuals = cisuber.get(keys=keys0) assert not actuals assert actuals == [] - assert sdb.cnt(keys0) == 0 + assert cisuber.cnt(keys0) == 0 - assert sdb.put(keys=keys0, vals=[saider0, saider1]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.put(keys=keys0, vals=[saider0, saider1]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said1 - assert sdb.add(keys=keys0, val=saider2) - assert sdb.cnt(keys0) == 3 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.add(keys=keys0, val=saider2) + assert cisuber.cnt(keys0) == 3 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said0, said1, said2] - actual = sdb.getLast(keys=keys0) + actual = cisuber.getLast(keys=keys0) assert actual.qb64 == said2 - assert sdb.pin(keys=keys0, vals=[saider1, saider2]) - assert sdb.cnt(keys0) == 2 - actuals = sdb.get(keys=keys0) # insertion order not lexicographic + assert cisuber.pin(keys=keys0, vals=[saider1, saider2]) + assert cisuber.cnt(keys0) == 2 + actuals = cisuber.get(keys=keys0) # insertion order not lexicographic assert len(actuals) == 2 sers = [actual.qb64 for actual in actuals] assert sers == [said1, said2] - assert sdb.put(keys=keys1, vals=[saider2, saider1, saider0]) - assert sdb.cnt(keys1) == 3 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.put(keys=keys1, vals=[saider2, saider1, saider0]) + assert cisuber.cnt(keys1) == 3 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic assert len(actuals) == 3 sers = [actual.qb64 for actual in actuals] assert sers == [said2, said1, said0] - assert sdb.rem(keys=keys1, val=saider1) - assert sdb.cnt(keys1) == 2 - actuals = sdb.get(keys=keys1) # insertion order not lexicographic + assert cisuber.rem(keys=keys1, val=saider1) + assert cisuber.cnt(keys1) == 2 + actuals = cisuber.get(keys=keys1) # insertion order not lexicographic sers = [actual.qb64 for actual in actuals] assert sers == [said2, said0] - sers = [val.qb64 for val in sdb.getIter(keys=keys1)] + sers = [val.qb64 for val in cisuber.getIter(keys=keys1)] assert sers == [said2, said0] - items = [(keys, val.qb64) for keys, val in sdb.getItemIter()] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter()] assert items == [ (keys1, said2), (keys1, said0), @@ -552,7 +1359,7 @@ def test_cesr_ioset_suber(): ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter()] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter()] assert items == [ ((*keys1, '00000000000000000000000000000000'), said2), ((*keys1, '00000000000000000000000000000002'), said0), @@ -560,38 +1367,30 @@ def test_cesr_ioset_suber(): ((*keys0, '00000000000000000000000000000001'), said2), ] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItem(keys=keys1)] - assert items == [ - ((*keys1, '00000000000000000000000000000000'), said2), - ((*keys1, '00000000000000000000000000000002'), said0), - ] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys0)] + assert items == [(keys0, said1), (keys0, said2)] - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoSetItemIter(keys=keys0)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getItemIter(keys=keys1)] assert items == [ - ((*keys0, '00000000000000000000000000000000'), said1), - ((*keys0, '00000000000000000000000000000001'), said2), + (keys1, said2), + (keys1, said0), ] topkeys = (seq1, "") - items = [(keys, val.qb64) for keys, val in sdb.getItemIter(keys=topkeys)] + items = [(keys, val.qb64) for keys, val in cisuber.getItemIter(keys=topkeys)] assert items == [ (keys1, said2), (keys1, said0), ] topkeys = (seq0, "") - items = [(iokeys, val.qb64) for iokeys, val in sdb.getIoItemIter(keys=topkeys)] + items = [(iokeys, val.qb64) for iokeys, val in cisuber.getFullItemIter(keys=topkeys)] assert items == [ ((*keys0, '00000000000000000000000000000000'), said1), ((*keys0, '00000000000000000000000000000001'), said2), ] - for iokeys, val in sdb.getIoItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 assert not os.path.exists(db.path) @@ -608,79 +1407,293 @@ def test_serder_suber(): assert db.name == "test" assert db.opened - sdb = subing.SerderSuber(db=db, subkey='bags.') - assert isinstance(sdb, subing.SerderSuber) - assert not sdb.sdb.flags()["dupsort"] + serber = subing.SerderSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" srdr0 = eventing.incept(keys=[pre]) keys = (pre, srdr0.said) - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + assert serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None - sdb.put(keys=keys, val=srdr0) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr0) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) - result = sdb.put(keys=keys, val=srdr1) + result = serber.put(keys=keys, val=srdr1) assert not result + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr0.said - result = sdb.pin(keys=keys, val=srdr1) + result = serber.pin(keys=keys, val=srdr1) assert result - actual = sdb.get(keys=keys) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said # test with keys as string not tuple keys = "{}.{}".format(pre, srdr1.said) - sdb.put(keys=keys, val=srdr1) - actual = sdb.get(keys=keys) + serber.put(keys=keys, val=srdr1) + actual = serber.get(keys=keys) assert isinstance(actual, serdering.SerderKERI) assert actual.said == srdr1.said - sdb.rem(keys) - actual = sdb.get(keys=keys) + assert serber.rem(keys) + actual = serber.get(keys=keys) assert actual is None # test missing entry at keys badkey = "badkey" - actual = sdb.get(badkey) + actual = serber.get(badkey) assert actual is None # test iteritems - sdb = subing.SerderSuber(db=db, subkey='pugs.') - assert isinstance(sdb, subing.SerderSuber) - sdb.put(keys=("a","1"), val=srdr0) - sdb.put(keys=("a","2"), val=srdr1) + serber = subing.SerderSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderSuber) + serber.put(keys=("a","1"), val=srdr0) + serber.put(keys=("a","2"), val=srdr1) - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter()] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] assert items == [(('a', '1'), srdr0.said), (('a', '2'), srdr1.said)] - assert sdb.put(keys=("b","1"), val=srdr0) - assert sdb.put(keys=("b","2"), val=srdr1) - assert sdb.put(keys=("bc","1"), val=srdr0) + assert serber.put(keys=("b","1"), val=srdr0) + assert serber.put(keys=("b","2"), val=srdr1) + assert serber.put(keys=("bc","1"), val=srdr0) topkeys = ("b", "") # append empty str to force trailing .sep - items = [(keys, srdr.said) for keys, srdr in sdb.getItemIter(keys=topkeys)] + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] assert items == [(('b', '1'), srdr0.said), (('b', '2'), srdr1.said)] assert not os.path.exists(db.path) assert not db.opened +def test_serder_ioset_suber(): + """ + Test SerderIoSetSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + serber = subing.SerderIoSetSuber(db=db, subkey='bags.') + assert isinstance(serber, subing.SerderIoSetSuber) + assert not serber.sdb.flags()["dupsort"] + assert serber.klas == serdering.SerderKERI + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + srdr0 = eventing.incept(keys=[pre]) + + keys = (pre, srdr0.said) + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + assert serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals # empty list + + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr0.said + + srdr1 = eventing.rotate(pre=pre, keys=[pre], dig=srdr0.said) + + result = serber.put(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr1.said + + + result = serber.pin(keys=keys, vals=(srdr1, )) + assert result + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + + # test rem a specific value + serber.put(keys=keys, vals=(srdr0, )) + actuals = serber.get(keys=keys) + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr1.said + assert isinstance(actuals[1], serdering.SerderKERI) + assert actuals[1].said == srdr0.said + + assert serber.rem(keys=keys, val=srdr1) + actuals = serber.get(keys=keys) + assert len(actuals) == 1 + assert isinstance(actuals[0], serdering.SerderKERI) + assert actuals[0].said == srdr0.said + + + # test with keys as string not tuple + keys = "{}.{}".format(pre, srdr1.said) + + serber.put(keys=keys, vals=(srdr1, )) + actuals = serber.get(keys=keys) + for actual in actuals: + assert isinstance(actual, serdering.SerderKERI) + assert actual.said == srdr1.said + + assert serber.rem(keys) + actuals = serber.get(keys=keys) + assert not actuals + + # test missing entry at keys + badkey = "badkey" + actuals = serber.get(badkey) + assert not actuals + + # test iteritems + serber = subing.SerderIoSetSuber(db=db, subkey='pugs.') + assert isinstance(serber, subing.SerderIoSetSuber) + serber.put(keys=("a","1"), vals=(srdr0, )) + serber.put(keys=("a","1"), vals=(srdr1, )) + serber.put(keys=("a","2"), vals=(srdr1, )) + + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter()] + assert items == [ + (('a', '1'), srdr0.said), + (('a', '1'), srdr1.said), + (('a', '2'), srdr1.said), + ] + + assert serber.put(keys=("b","1"), vals=(srdr0, )) + assert serber.put(keys=("b","2"), vals=(srdr1, )) + assert serber.put(keys=("bc","1"), vals=(srdr0, )) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in serber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), srdr0.said), + (('b', '2'), srdr1.said)] + + assert not os.path.exists(db.path) + assert not db.opened + + +def test_schemer_suber(): + """ + Test SchemerSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + scmber = subing.SchemerSuber(db=db, subkey='bags.') + assert isinstance(scmber, subing.SchemerSuber) + assert not scmber.sdb.flags()["dupsort"] + assert scmber.klas == scheming.Schemer + + pre = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + + raw0 = (b'{"$id":"EMRvS7lGxc1eDleXBkvSHkFs8vUrslRcla6UXOJdcczw","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"str' + b'ing"},"b":{"type":"number"},"c":{"type":"string","format":"date-time"}}}') + + scmr0 = scheming.Schemer(raw=raw0) + + keys = (pre, scmr0.said) + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + scmber.put(keys=keys, val=scmr0) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + raw1 = (b'{"$id":"ENQKl3r1Z6HiLXOD-050aVvKziCWJtXWg3vY2FWUGSxG","$schema":"http://json' + b'-schema.org/draft-07/schema#","type":"object","properties":{"a":{"type":"obj' + b'ect","properties":{"b":{"type":"number"},"c":{"type":"string","format":"date' + b'-time"}}}}}') + scmr1 = scheming.Schemer(raw=raw1) + + result = scmber.put(keys=keys, val=scmr1) + assert not result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr0.said + + result = scmber.pin(keys=keys, val=scmr1) + assert result + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + # test with keys as string not tuple + keys = "{}.{}".format(pre, scmr1.said) + + scmber.put(keys=keys, val=scmr1) + actual = scmber.get(keys=keys) + assert isinstance(actual, scheming.Schemer) + assert actual.said == scmr1.said + + assert scmber.rem(keys) + actual = scmber.get(keys=keys) + assert actual is None + + # test missing entry at keys + badkey = "badkey" + actual = scmber.get(badkey) + assert actual is None + + # test iteritems + scmber = subing.SchemerSuber(db=db, subkey='pugs.') + assert isinstance(scmber, subing.SchemerSuber) + scmber.put(keys=("a","1"), val=scmr0) + scmber.put(keys=("a","2"), val=scmr1) + + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter()] + assert items == [(('a', '1'), scmr0.said), + (('a', '2'), scmr1.said)] + + assert scmber.put(keys=("b","1"), val=scmr0) + assert scmber.put(keys=("b","2"), val=scmr1) + assert scmber.put(keys=("bc","1"), val=scmr0) + + topkeys = ("b", "") # append empty str to force trailing .sep + items = [(keys, srdr.said) for keys, srdr in scmber.getItemIter(keys=topkeys)] + assert items == [(('b', '1'), scmr0.said), + (('b', '2'), scmr1.said)] + + with pytest.raises(TypeError): + scmber = subing.SchemerSuber(db=db, subkey='bags.', klas=coring.Matter) + + + assert not os.path.exists(db.path) + assert not db.opened def test_cesr_suber(): @@ -809,15 +1822,15 @@ def test_cesr_suber(): (('b', '2'), val1.qb64)] # Try Siger Indexer Subclass - sdb = subing.CesrSuber(db=db, subkey='pigs.', klas=coring.Siger) + sdb = subing.CesrSuber(db=db, subkey='pigs.', klas=indexing.Siger) assert isinstance(sdb, subing.CesrSuber) - assert issubclass(sdb.klas, coring.Siger) + assert issubclass(sdb.klas, indexing.Siger) sig0 = 'AACdI8OSQkMJ9r-xigjEByEjIua7LHH3AOJ22PQKqljMhuhcgh9nGRcKnsz5KvKd7K_H9-1298F4Id1DxvIoEmCQ' - val0 = coring.Siger(qb64=sig0) + val0 = indexing.Siger(qb64=sig0) keys = ("zeta", "cat") assert sdb.put(keys=keys, val=val0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Siger) + assert isinstance(actual, indexing.Siger) assert actual.qb64 == val0.qb64 @@ -825,10 +1838,117 @@ def test_cesr_suber(): assert not db.opened """Done Test""" +def test_cesr_on_suber(): + """ + Test CesrOnSuber LMDBer sub database class + """ + + with dbing.openLMDB() as db: + assert isinstance(db, dbing.LMDBer) + assert db.name == "test" + assert db.opened + + onsuber = subing.CesrOnSuber(db=db, subkey='bags.') + assert isinstance(onsuber, subing.CesrOnSuber) + assert not onsuber.sdb.flags()["dupsort"] + + prew = "BDzwEHHzq7K0gzQPYGGwTmuupUhPx5_yZ-Wk1x4ejhcc" + w = coring.Matter(qb64=prew) + prex = "BHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + x = coring.Matter(qb64=prex) + prey = "EHHzqZWzwE-Wk7K0gzQPYGGwTmuupUhPx5_y1x4ejhcc" + y = coring.Matter(qb64=prey) + prez = "EAPYGGwTmuupWzwEHHzq7K0gzUhPx5_yZ-Wk1x4ejhcc" + z = coring.Matter(qb64=prez) + + + # test append + assert 0 == onsuber.appendOn(keys=("a",), val=w) + assert 1 == onsuber.appendOn(keys=("a",), val=x) + assert 2 == onsuber.appendOn(keys=("a",), val=y) + assert 3 == onsuber.appendOn(keys=("a",), val=z) + + assert onsuber.cntOn(keys=("a",)) == 4 + assert onsuber.cntOn(keys=("a",), on=2) == 2 + assert onsuber.cntOn(keys=("a",), on=4) == 0 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='a', on=2)] + assert items == [(('a',), 2, y.qb64), + (('a',), 3, z.qb64)] + + assert 0 == onsuber.appendOn(keys=("ac",), val=z) + assert 0 == onsuber.appendOn(keys=("b",), val=w) + assert 1 == onsuber.appendOn(keys=("b",), val=x) + assert 0 == onsuber.appendOn(keys=("bc",), val=y) + + + assert onsuber.cntOn(keys=("b",)) == 2 + assert onsuber.cntOn(keys=("ac",), on=2) == 0 + assert onsuber.cntOn(keys="") == 8 + + items = [(keys, val.qb64) for keys, val in onsuber.getItemIter()] + assert items == [(('a', '00000000000000000000000000000000'), w.qb64), + (('a', '00000000000000000000000000000001'), x.qb64), + (('a', '00000000000000000000000000000002'), y.qb64), + (('a', '00000000000000000000000000000003'), z.qb64), + (('ac', '00000000000000000000000000000000'), z.qb64), + (('b', '00000000000000000000000000000000'), w.qb64), + (('b', '00000000000000000000000000000001'), x.qb64), + (('bc', '00000000000000000000000000000000'), y.qb64)] + + # test getOnItemIter + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='b')] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys=('b', ))] + assert items == [(('b',), 0, w.qb64), + (('b',), 1, x.qb64)] + + items = [item for item in onsuber.getOnItemIter(keys=('b', ""))] + assert items == [] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter(keys='')] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + items = [(keys, on, val.qb64) for keys, on, val in onsuber.getOnItemIter()] + assert items == [(('a',), 0, w.qb64), + (('a',), 1, x.qb64), + (('a',), 2, y.qb64), + (('a',), 3, z.qb64), + (('ac',), 0, z.qb64), + (('b',), 0, w.qb64), + (('b',), 1, x.qb64), + (('bc',), 0, y.qb64)] + + assert not os.path.exists(db.path) + assert not db.opened + -def test_cat_suber(): + +def test_cat_cesr_suber(): """ - Test CatSuber LMDBer sub database class + Test CatCesrSuber LMDBer sub database class """ with dbing.openLMDB() as db: @@ -947,15 +2067,15 @@ def test_cat_suber(): assert isinstance(val, klas) # Try Siger Indexer Subclass - sdb = subing.CatCesrSuber(db=db, subkey='pigs.', klas=(coring.Siger, )) + sdb = subing.CatCesrSuber(db=db, subkey='pigs.', klas=(indexing.Siger, )) assert isinstance(sdb, subing.CatCesrSuber) - assert issubclass(sdb.klas[0], coring.Siger) + assert issubclass(sdb.klas[0], indexing.Siger) sig0 = 'AACdI8OSQkMJ9r-xigjEByEjIua7LHH3AOJ22PQKqljMhuhcgh9nGRcKnsz5KvKd7K_H9-1298F4Id1DxvIoEmCQ' - val0 = coring.Siger(qb64=sig0) + val0 = indexing.Siger(qb64=sig0) keys = ("zeta", "cat") assert sdb.put(keys=keys, val=[val0]) actual = sdb.get(keys=keys) - assert isinstance(actual[0], coring.Siger) + assert isinstance(actual[0], indexing.Siger) assert actual[0].qb64 == val0.qb64 assert not os.path.exists(db.path) @@ -963,7 +2083,7 @@ def test_cat_suber(): """Done Test""" -def test_cat__cesr_ioset_suber(): +def test_cat_cesr_ioset_suber(): """ Test CatIoSetSuber LMDBer sub database class """ @@ -1109,7 +2229,7 @@ def test_cat__cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter()] + for iokeys, vals in sdb.getFullItemIter()] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), @@ -1118,15 +2238,15 @@ def test_cat__cesr_ioset_suber(): (keys2 + ('00000000000000000000000000000001', ), [sqr2.qb64, dgr2.qb64]) ] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItem(keys=keys1)] - assert items == [(keys1 + ('00000000000000000000000000000000', ), [sqr2.qb64, dgr2.qb64])] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getItemIter(keys=keys1)] + assert items == [(keys1, [sqr2.qb64, dgr2.qb64])] - items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoSetItemIter(keys=keys0)] + items = [(keys, [val.qb64 for val in vals]) + for keys, vals in sdb.getItemIter(keys=keys0)] assert items == [ - (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), - (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), + (keys0, [sqr0.qb64, dgr0.qb64]), + (keys0, [sqr1.qb64, dgr1.qb64]), ] @@ -1139,30 +2259,24 @@ def test_cat__cesr_ioset_suber(): ] items = [(iokeys, [val.qb64 for val in vals]) - for iokeys, vals in sdb.getIoItemIter(keys=topkeys)] + for iokeys, vals in sdb.getFullItemIter(keys=topkeys)] assert items == [ (keys0 + ('00000000000000000000000000000000', ), [sqr0.qb64, dgr0.qb64]), (keys0 + ('00000000000000000000000000000001', ), [sqr1.qb64, dgr1.qb64]), ] - for iokeys, val in sdb.getIoItemIter(): - assert sdb.remIokey(iokeys=iokeys) - - assert sdb.cnt(keys=keys0) == 0 - assert sdb.cnt(keys=keys1) == 0 - assert sdb.cnt(keys=keys2) == 0 # Try Siger Indexer Subclass - sdb = subing.CatCesrIoSetSuber(db=db, subkey='pigs.', klas=(coring.Siger, )) + sdb = subing.CatCesrIoSetSuber(db=db, subkey='pigs.', klas=(indexing.Siger, )) assert isinstance(sdb, subing.CatCesrIoSetSuber) - assert issubclass(sdb.klas[0], coring.Siger) + assert issubclass(sdb.klas[0], indexing.Siger) sig0 = 'AACdI8OSQkMJ9r-xigjEByEjIua7LHH3AOJ22PQKqljMhuhcgh9nGRcKnsz5KvKd7K_H9-1298F4Id1DxvIoEmCQ' - val0 = coring.Siger(qb64=sig0) + val0 = indexing.Siger(qb64=sig0) keys = ("zeta", "cat") assert sdb.put(keys=keys, vals=[[val0]]) actuals = sdb.get(keys=keys) - assert isinstance(actuals[0][0], coring.Siger) + assert isinstance(actuals[0][0], indexing.Siger) assert actuals[0][0].qb64 == val0.qb64 assert not os.path.exists(db.path) @@ -1291,15 +2405,15 @@ def test_cesr_dup_suber(): assert pres == [val2.qb64, val1.qb64] # lexi order # Try Siger Indexer Subclass - sdb = subing.CesrDupSuber(db=db, subkey='pigs.', klas=coring.Siger) + sdb = subing.CesrDupSuber(db=db, subkey='pigs.', klas=indexing.Siger) assert isinstance(sdb, subing.CesrDupSuber) - assert issubclass(sdb.klas, coring.Siger) + assert issubclass(sdb.klas, indexing.Siger) sig0 = 'AACdI8OSQkMJ9r-xigjEByEjIua7LHH3AOJ22PQKqljMhuhcgh9nGRcKnsz5KvKd7K_H9-1298F4Id1DxvIoEmCQ' - val0 = coring.Siger(qb64=sig0) + val0 = indexing.Siger(qb64=sig0) keys = ("zeta", "cat") assert sdb.put(keys=keys, vals=[val0]) actuals = sdb.get(keys=keys) - assert isinstance(actuals[0], coring.Siger) + assert isinstance(actuals[0], indexing.Siger) assert actuals[0].qb64 == val0.qb64 @@ -1319,13 +2433,13 @@ def test_signer_suber(): sdb = subing.SignerSuber(db=db, subkey='bags.') # default klas is Signer assert isinstance(sdb, subing.SignerSuber) - assert issubclass(sdb.klas, coring.Signer) + assert issubclass(sdb.klas, core.Signer) assert not sdb.sdb.flags()["dupsort"] # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) seed0 = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' b'\xde\x06\xc0+') - signer0 = coring.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) + signer0 = core.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) assert signer0.verfer.code == coring.MtrDex.Ed25519 assert signer0.verfer.transferable # default assert signer0.qb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' @@ -1335,7 +2449,7 @@ def test_signer_suber(): seed1 = (b'`\x05\x93\xb9\x9b6\x1e\xe0\xd7\x98^\x94\xc8Et\xf2\xc4\xcd\x94\x18' b'\xc6\xae\xb9\xb6m\x12\xc4\x80\x03\x07\xfc\xf7') - signer1 = coring.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) + signer1 = core.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) assert signer1.verfer.code == coring.MtrDex.Ed25519 assert signer1.verfer.transferable # default assert signer1.qb64b == b'AGAFk7mbNh7g15helMhFdPLEzZQYxq65tm0SxIADB_z3' @@ -1344,7 +2458,7 @@ def test_signer_suber(): keys = (signer0.verfer.qb64, ) # must be verfer as key to get transferable sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1354,14 +2468,14 @@ def test_signer_suber(): sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 # try put different val when already put result = sdb.put(keys=keys, val=signer1) assert not result - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1370,7 +2484,7 @@ def test_signer_suber(): result = sdb.pin(keys=keys, val=signer1) assert result actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer1.qb64 assert actual.verfer.qb64 == signer1.verfer.qb64 @@ -1379,7 +2493,7 @@ def test_signer_suber(): sdb.pin(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1422,7 +2536,7 @@ def test_crypt_signer_suber(): # preseed = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) seed0 = (b'\x18;0\xc4\x0f*vF\xfa\xe3\xa2Eee\x1f\x96o\xce)G\x85\xe3X\x86\xda\x04\xf0\xdc' b'\xde\x06\xc0+') - signer0 = coring.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) + signer0 = core.Signer(raw=seed0, code=coring.MtrDex.Ed25519_Seed) assert signer0.verfer.code == coring.MtrDex.Ed25519 assert signer0.verfer.transferable # default assert signer0.qb64b == b'ABg7MMQPKnZG-uOiRWVlH5ZvzilHheNYhtoE8NzeBsAr' @@ -1432,7 +2546,7 @@ def test_crypt_signer_suber(): seed1 = (b'`\x05\x93\xb9\x9b6\x1e\xe0\xd7\x98^\x94\xc8Et\xf2\xc4\xcd\x94\x18' b'\xc6\xae\xb9\xb6m\x12\xc4\x80\x03\x07\xfc\xf7') - signer1 = coring.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) + signer1 = core.Signer(raw=seed1, code=coring.MtrDex.Ed25519_Seed) assert signer1.verfer.code == coring.MtrDex.Ed25519 assert signer1.verfer.transferable # default assert signer1.qb64b == b'AGAFk7mbNh7g15helMhFdPLEzZQYxq65tm0SxIADB_z3' @@ -1441,27 +2555,27 @@ def test_crypt_signer_suber(): # rawsalt =pysodium.randombytes(pysodium.crypto_pwhash_SALTBYTES) rawsalt = b'0123456789abcdef' - salter = coring.Salter(raw=rawsalt) + salter = core.Salter(raw=rawsalt) salt = salter.qb64 assert salt == '0AAwMTIzNDU2Nzg5YWJjZGVm' stem = "blue" # cryptseed0 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed0 = b'h,#|\x8ap"\x12\xc43t2\xa6\xe1\x18\x19\xf0f2,y\xc4\xc21@\xf5@\x15.\xa2\x1a\xcf' - cryptsigner0 = coring.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, + cryptsigner0 = core.Signer(raw=cryptseed0, code=coring.MtrDex.Ed25519_Seed, transferable=False) seed0 = cryptsigner0.qb64 aeid0 = cryptsigner0.verfer.qb64 assert aeid0 == 'BCa7mK96FwxkU0TdF54Yqg3qBDXUWpOhQ_Mtr7E77yZB' - decrypter = coring.Decrypter(seed=seed0) - encrypter = coring.Encrypter(verkey=aeid0) + decrypter = core.Decrypter(seed=seed0) + encrypter = core.Encrypter(verkey=aeid0) assert encrypter.verifySeed(seed=seed0) # cryptseed1 = pysodium.randombytes(pysodium.crypto_sign_SEEDBYTES) cryptseed1 = (b"\x89\xfe{\xd9'\xa7\xb3\x89#\x19\xbec\xee\xed\xc0\xf9\x97\xd0\x8f9\x1dyNI" b'I\x98\xbd\xa4\xf6\xfe\xbb\x03') - cryptsigner1 = coring.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, + cryptsigner1 = core.Signer(raw=cryptseed1, code=coring.MtrDex.Ed25519_Seed, transferable=False) @@ -1472,14 +2586,14 @@ def test_crypt_signer_suber(): sdb = subing.CryptSignerSuber(db=db, subkey='bags.') # default klas is Signer assert isinstance(sdb, subing.CryptSignerSuber) - assert issubclass(sdb.klas, coring.Signer) + assert issubclass(sdb.klas, core.Signer) assert not sdb.sdb.flags()["dupsort"] # Test without encrypter or decrypter keys = (signer0.verfer.qb64, ) # must be verfer as key to get transferable sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1489,14 +2603,14 @@ def test_crypt_signer_suber(): sdb.put(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 # try put different val when already put result = sdb.put(keys=keys, val=signer1) assert not result - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1505,7 +2619,7 @@ def test_crypt_signer_suber(): result = sdb.pin(keys=keys, val=signer1) assert result actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer1.qb64 assert actual.verfer.qb64 == signer1.verfer.qb64 @@ -1514,7 +2628,7 @@ def test_crypt_signer_suber(): sdb.pin(keys=keys, val=signer0) actual = sdb.get(keys=keys) - assert isinstance(actual, coring.Signer) + assert isinstance(actual, core.Signer) assert actual.qb64 == signer0.qb64 assert actual.verfer.qb64 == signer0.verfer.qb64 @@ -1537,8 +2651,8 @@ def test_crypt_signer_suber(): # now test with encrypter and decrypter - encrypter0 = coring.Encrypter(verkey=cryptsigner0.verfer.qb64) - decrypter0 = coring.Decrypter(seed=cryptsigner0.qb64b) + encrypter0 = core.Encrypter(verkey=cryptsigner0.verfer.qb64) + decrypter0 = core.Decrypter(seed=cryptsigner0.qb64b) # first pin with encrypter assert sdb.pin(keys=signer0.verfer.qb64b, val=signer0, encrypter=encrypter0) @@ -1546,12 +2660,12 @@ def test_crypt_signer_suber(): # now get actual0 = sdb.get(keys=signer0.verfer.qb64b, decrypter=decrypter0) - assert isinstance(actual0, coring.Signer) + assert isinstance(actual0, core.Signer) assert actual0.qb64 == signer0.qb64 assert actual0.verfer.qb64 == signer0.verfer.qb64 actual1 = sdb.get(keys=signer1.verfer.qb64b, decrypter=decrypter0) - assert isinstance(actual1, coring.Signer) + assert isinstance(actual1, core.Signer) assert actual1.qb64 == signer1.qb64 assert actual1.verfer.qb64 == signer1.verfer.qb64 @@ -1588,8 +2702,8 @@ def test_crypt_signer_suber(): # test re-encrypt - encrypter1 = coring.Encrypter(verkey=cryptsigner1.verfer.qb64) - decrypter1 = coring.Decrypter(seed=cryptsigner1.qb64b) + encrypter1 = core.Encrypter(verkey=cryptsigner1.verfer.qb64) + decrypter1 = core.Decrypter(seed=cryptsigner1.qb64b) for keys, sgnr in sdb.getItemIter(decrypter=decrypter0): sdb.pin(keys, sgnr, encrypter=encrypter1) @@ -1617,15 +2731,15 @@ def test_crypt_signer_suber(): assert manager.algo == keeping.Algos.salty assert manager.salt == salt # encrypted on disk but property decrypts if seed assert manager.pidx == 0 - assert manager.tier == coring.Tiers.low - saltCipher0 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + assert manager.tier == core.Tiers.low + saltCipher0 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert saltCipher0.decrypt(seed=seed0).qb64 == salt manager.updateAeid(aeid=cryptsigner1.verfer.qb64, seed=cryptsigner1.qb64) assert manager.aeid == cryptsigner1.verfer.qb64 == 'BEcOrMrG_7r_NWaLl6h8UJapwIfQWIkjrIPXkCZm2fFM' assert manager.salt == salt - saltCipher1 = coring.Cipher(qb64=manager.ks.gbls.get('salt')) + saltCipher1 = core.Cipher(qb64=manager.ks.gbls.get('salt')) assert not saltCipher0.qb64 == saltCipher1.qb64 # old cipher different """End Test""" @@ -1633,11 +2747,21 @@ def test_crypt_signer_suber(): if __name__ == "__main__": - test_cesr_ioset_suber() - test_serder_suber() + test_suber() + test_on_suber() + test_B64_suber() + test_dup_suber() + test_iodup_suber() + test_on_iodup_suber() + test_ioset_suber() + test_cat_cesr_suber() test_cesr_suber() - test_cat_suber() - test_cat__cesr_ioset_suber() + test_cesr_on_suber() + test_cesr_ioset_suber() + test_cat_cesr_ioset_suber() test_cesr_dup_suber() + test_serder_suber() + test_serder_ioset_suber() + test_schemer_suber() test_signer_suber() test_crypt_signer_suber() diff --git a/tests/demo/test_demo.py b/tests/demo/test_demo.py index ac4303dd7..f0c8e4e7c 100644 --- a/tests/demo/test_demo.py +++ b/tests/demo/test_demo.py @@ -11,8 +11,12 @@ from hio.core.tcp import clienting, serving from keri import help # logger support -from keri.app import habbing, directing + +from keri import core from keri.core import eventing, coring + +from keri.app import habbing, directing + from keri.demo import demoing @@ -25,7 +29,7 @@ def test_direct_mode_bob_eve_demo(): raw = b"raw salt to test" # create bob signers and secrecies - bobSigners = coring.Salter(raw=raw).signers(count=8, path="bob", temp=True) + bobSigners = core.Salter(raw=raw).signers(count=8, path="bob", temp=True) bobSecrecies = [[signer.qb64] for signer in bobSigners] # bob inception transferable (nxt digest not empty) @@ -37,7 +41,7 @@ def test_direct_mode_bob_eve_demo(): assert bob == 'EFa1wAk_coghxxGCID6jEN79Kmvyj0Y1wWN_ndUv3LjW' # create eve signers and secrecies - eveSigners = coring.Salter(raw=raw).signers(count=8, path="eve", temp=True) + eveSigners = core.Salter(raw=raw).signers(count=8, path="eve", temp=True) eveSecrecies = [[signer.qb64] for signer in eveSigners] # eve inception transferable (nxt digest not empty) @@ -156,7 +160,7 @@ def test_direct_mode_sam_eve_demo(): raw = b"raw salt to test" # create sam signers and secrecies - samSigners = coring.Salter(raw=raw).signers(count=8, path="sam", temp=True) + samSigners = core.Salter(raw=raw).signers(count=8, path="sam", temp=True) samSecrecies = [[signer.qb64] for signer in samSigners] # sam inception transferable (nxt digest not empty) @@ -168,7 +172,7 @@ def test_direct_mode_sam_eve_demo(): assert sam == 'EDkU2U_TPKca14VElEItpj7twohQL60GIaUPvSHAghga' # create eve signers and secrecies - eveSigners = coring.Salter(raw=raw).signers(count=8, path="eve", temp=True) + eveSigners = core.Salter(raw=raw).signers(count=8, path="eve", temp=True) eveSecrecies = [[signer.qb64] for signer in eveSigners] # eve inception transferable (nxt digest not empty) @@ -309,7 +313,7 @@ def test_run_bob_eve_demo(): # create bob secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="bob", temp=True)] @@ -325,7 +329,7 @@ def test_run_bob_eve_demo(): # create eve secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="eve", temp=True)] @@ -363,7 +367,7 @@ def test_run_sam_eve_demo(): # create sam secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="sam", temp=True)] @@ -380,7 +384,7 @@ def test_run_sam_eve_demo(): # create eve secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="eve", temp=True)] @@ -410,19 +414,19 @@ def test_indirect_mode_sam_cam_wit_demo(): raw = b"raw salt to test" # create sam signers and secrecies - samSigners = coring.Salter(raw=raw).signers(count=8, path="sam", temp=True) + samSigners = core.Salter(raw=raw).signers(count=8, path="sam", temp=True) samSecrecies = [[signer.qb64] for signer in samSigners] # create cam signers and secrecies - camSigners = coring.Salter(raw=raw).signers(count=8, path="cam", temp=True) + camSigners = core.Salter(raw=raw).signers(count=8, path="cam", temp=True) camSecrecies = [[signer.qb64] for signer in camSigners] - with (habbing.openHby(name="cam", base="test") as camHby, - habbing.openHby(name="sam", base="test") as samHby, - habbing.openHby(name="wit", base="test") as witHby): + with (habbing.openHby(name="cam", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as camHby, + habbing.openHby(name="sam", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as samHby, + habbing.openHby(name="wit", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as witHby): samPort = 5620 # sam's TCP listening port for server - witPort = 5621 # wit' TCP listneing port for server + witPort = 5621 # wit' TCP listening port for server # setup the witness witHab = witHby.makeHab(name="Wit", diff --git a/tests/end/test_ending.py b/tests/end/test_ending.py index 9b62a0b36..4e90bad64 100644 --- a/tests/end/test_ending.py +++ b/tests/end/test_ending.py @@ -13,8 +13,12 @@ from hio.help import Hict from keri import help, kering -from keri.app import habbing + +from keri import core from keri.core import coring, serdering + +from keri.app import habbing + from keri.end import ending logger = help.ogler.getLogger() @@ -35,13 +39,13 @@ def test_mimes(): assert ending.KeriMimes.cesr == 'application/keri+cesr' # Usage: to get Mime from serialization kind - assert getattr(ending.Mimes, coring.Serials.json.lower()) == ending.Mimes.json - assert getattr(ending.Mimes, coring.Serials.mgpk.lower()) == ending.Mimes.mgpk - assert getattr(ending.Mimes, coring.Serials.cbor.lower()) == ending.Mimes.cbor + assert getattr(ending.Mimes, coring.Kinds.json.lower()) == ending.Mimes.json + assert getattr(ending.Mimes, coring.Kinds.mgpk.lower()) == ending.Mimes.mgpk + assert getattr(ending.Mimes, coring.Kinds.cbor.lower()) == ending.Mimes.cbor - assert getattr(ending.KeriMimes, coring.Serials.json.lower()) == ending.KeriMimes.json - assert getattr(ending.KeriMimes, coring.Serials.mgpk.lower()) == ending.KeriMimes.mgpk - assert getattr(ending.KeriMimes, coring.Serials.cbor.lower()) == ending.KeriMimes.cbor + assert getattr(ending.KeriMimes, coring.Kinds.json.lower()) == ending.KeriMimes.json + assert getattr(ending.KeriMimes, coring.Kinds.mgpk.lower()) == ending.KeriMimes.mgpk + assert getattr(ending.KeriMimes, coring.Kinds.cbor.lower()) == ending.KeriMimes.cbor """Done Test""" @@ -59,7 +63,7 @@ def test_signature_designature(): # db = basing.Baser(name=name, temp=temp, reopen=reopen) # Setup Habery and Hab - with habbing.openHby(name=name, base=base) as hby: + with habbing.openHby(name=name, base=base, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: # hby = habbing.Habery(name=name, base=base, temp=temp, free=True) hab = hby.makeHab(name=name, icount=3) print() @@ -313,7 +317,7 @@ def test_seid_api(): # Setup Habery and Hab name = 'zoe' base = 'test' - with habbing.openHby(name=name, base=base) as hby: + with habbing.openHby(name=name, base=base, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name=name) # hab = setupTestHab(name='zoe') # must do it here to inject into Falcon endpoint resource instances @@ -382,7 +386,7 @@ def test_get_admin(): # Setup Habery and Hab name = 'zoe' base = 'test' - with habbing.openHby(name=name, base=base) as hby: + with habbing.openHby(name=name, base=base, salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name=name) # hab = setupTestHab(name='zoe') @@ -407,7 +411,7 @@ def test_get_oobi(): # Setup Habery and Hab name = 'oobi' base = 'test' - salt = coring.Salter(raw=b'0123456789abcdef').qb64 + salt = core.Salter(raw=b'0123456789abcdef').qb64 with habbing.openHby(name=name, base=base, salt=salt) as hby: hab = hby.makeHab(name=name) msgs = bytearray() @@ -441,7 +445,7 @@ def test_get_oobi(): hab = hby.makeHab(name=name, delpre=delhab.pre) assert hab.pre == "EPERMS4wKU7ejhCdhI2qQR8snEx1cislR9C9bSEs0kS5" - assert hab.kever.delegator == delhab.pre + assert hab.kever.delpre == delhab.pre msgs.extend(hab.makeEndRole(eid=hab.pre, role=kering.Roles.controller, @@ -482,7 +486,7 @@ def test_get_oobi(): def test_siginput(mockHelpingNowUTC): print() - with habbing.openHab(name="test", base="test", temp=True) as (hby, hab): + with habbing.openHab(name="test", base="test", temp=True, salt=b'0123456789abcdef') as (hby, hab): headers = Hict([ ("Content-Type", "application/json"), ("Content-Length", "256"), diff --git a/tests/help/test_helping.py b/tests/help/test_helping.py index 23779a74d..4d5d7baae 100644 --- a/tests/help/test_helping.py +++ b/tests/help/test_helping.py @@ -15,6 +15,8 @@ from keri.help.helping import isign, sceil from keri.help.helping import extractValues from keri.help.helping import dictify, datify, klasify +from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) def test_utilities(): """ @@ -273,5 +275,172 @@ def test_iso8601(): """ End Test """ + +def test_b64_conversions(): + """ + Test Base64 conversion utility routines + """ + + cs = intToB64(0) + assert cs == "A" + i = b64ToInt(cs) + assert i == 0 + + cs = intToB64(0, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = b64ToInt(cs) + + cs = intToB64(None, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = b64ToInt(cs) + + cs = intToB64b(0) + assert cs == b"A" + i = b64ToInt(cs) + assert i == 0 + + cs = intToB64(27) + assert cs == "b" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64b(27) + assert cs == b"b" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64(27, l=2) + assert cs == "Ab" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64b(27, l=2) + assert cs == b"Ab" + i = b64ToInt(cs) + assert i == 27 + + cs = intToB64(80) + assert cs == "BQ" + i = b64ToInt(cs) + assert i == 80 + + cs = intToB64b(80) + assert cs == b"BQ" + i = b64ToInt(cs) + assert i == 80 + + cs = intToB64(4095) + assert cs == '__' + i = b64ToInt(cs) + assert i == 4095 + + cs = intToB64b(4095) + assert cs == b'__' + i = b64ToInt(cs) + assert i == 4095 + + cs = intToB64(4096) + assert cs == 'BAA' + i = b64ToInt(cs) + assert i == 4096 + + cs = intToB64b(4096) + assert cs == b'BAA' + i = b64ToInt(cs) + assert i == 4096 + + cs = intToB64(6011) + assert cs == "Bd7" + i = b64ToInt(cs) + assert i == 6011 + + cs = intToB64b(6011) + assert cs == b"Bd7" + i = b64ToInt(cs) + assert i == 6011 + + s = "-BAC" + b = codeB64ToB2(s[:]) + assert len(b) == 3 + assert b == b'\xf8\x10\x02' + t = codeB2ToB64(b, 4) + assert t == s[:] + i = int.from_bytes(b, 'big') + assert i == 0o76010002 + i >>= 2 * (len(s) % 4) + assert i == 0o76010002 + p = nabSextets(b, 4) + assert p == b'\xf8\x10\x02' + + b = codeB64ToB2(s[:3]) + assert len(b) == 3 + assert b == b'\xf8\x10\x00' + t = codeB2ToB64(b, 3) + assert t == s[:3] + i = int.from_bytes(b, 'big') + assert i == 0o76010000 + i >>= 2 * (len(s[:3]) % 4) + assert i == 0o760100 + p = nabSextets(b, 3) + assert p == b'\xf8\x10\x00' + + b = codeB64ToB2(s[:2]) + assert len(b) == 2 + assert b == b'\xf8\x10' + t = codeB2ToB64(b, 2) + assert t == s[:2] + i = int.from_bytes(b, 'big') + assert i == 0o174020 + i >>= 2 * (len(s[:2]) % 4) + assert i == 0o7601 + p = nabSextets(b, 2) + assert p == b'\xf8\x10' + + b = codeB64ToB2(s[:1]) + assert len(b) == 1 + assert b == b'\xf8' + t = codeB2ToB64(b, 1) + assert t == s[:1] + i = int.from_bytes(b, 'big') + assert i == 0o370 + i >>= 2 * (len(s[:1]) % 4) + assert i == 0o76 + p = nabSextets(b, 1) + assert p == b'\xf8' + + assert B64_CHARS == ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', + 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_') + assert '@' not in B64_CHARS + assert 'A' in B64_CHARS + + text = b"-A-Bg-1-3-cd" + match = Reb64.match(text) + assert match + assert match is not None + + text = b'' + match = Reb64.match(text) + assert match + assert match is not None + + text = b'123#$' + match = Reb64.match(text) + assert not match + assert match is None + + """End Test""" + + if __name__ == "__main__": + test_utilities() + test_datify() + test_dictify() test_klasify() + test_extractvalues() + test_iso8601() + test_b64_conversions() diff --git a/tests/help/test_ogling.py b/tests/help/test_ogling.py index 55ff5521c..78cd3777d 100644 --- a/tests/help/test_ogling.py +++ b/tests/help/test_ogling.py @@ -8,7 +8,7 @@ import os import logging -from keri.help import ogling +from hio.help import ogling from keri import help diff --git a/tests/peer/test_exchanging.py b/tests/peer/test_exchanging.py index 3c555bb12..50be8ec62 100644 --- a/tests/peer/test_exchanging.py +++ b/tests/peer/test_exchanging.py @@ -3,8 +3,15 @@ tests.peer.test_exchanging module """ +import pysodium +import pytest + +from keri import kering +from keri import core +from keri.core import coring, serdering, MtrDex, parsing + from keri.app import habbing, forwarding, storing, signing -from keri.core import coring, serdering + from keri.peer import exchanging from keri.vdr.eventing import incept @@ -43,7 +50,89 @@ def test_nesting(): assert pathed == {} +def test_essrs(): + with habbing.openHab(name="sid", base="test", salt=b'0123456789abcdef') as (hby, hab), \ + habbing.openHab(name="rec", base="test", salt=b'0123456789abcdef') as (recHby, recHab): + + ims = hab.makeOwnInception() + parsing.Parser().parse(ims=ims, kvy=recHby.kvy) + # create the test message with essr attachment + msg = "This is a test message that must be secured" + rkever = recHab.kever + pubkey = pysodium.crypto_sign_pk_to_box_pk(rkever.verfers[0].raw) + raw = pysodium.crypto_box_seal(msg.encode("utf-8"), pubkey) + + texter = coring.Texter(raw=raw) + diger = coring.Diger(ser=raw, code=MtrDex.Blake3_256) + essr, _ = exchanging.exchange(route='/essr/req', sender=hab.pre, diger=diger, + modifiers=dict(src=hab.pre, dest=recHab.pre)) + ims = hab.endorse(serder=essr, pipelined=False) + ims.extend(core.Counter(core.Codens.ESSRPayloadGroup, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) + ims.extend(texter.qb64b) + + exc = exchanging.Exchanger(hby=recHby, handlers=[]) + parsing.Parser().parse(ims=ims, + kvy=recHby.kvy, + exc=exc, + gvrsn=kering.Vrsn_1_0) # parser does not support version2 count codes + + # Pull the logged exn and verify the attributes digest matches the attachment + serder = recHby.db.exns.get(keys=(essr.said,)) + assert serder.ked['a'] == diger.qb64 + + # Pull the logged ESSR attachment and verify it is the one attached + texter = recHby.db.essrs.get(keys=(serder.said,)) + raw = recHab.decrypt(ser=texter[0].raw) + assert raw.decode("utf-8") == msg + + # Test with invalid diger + diger = coring.Diger(qb64="EKC8085pwSwzLwUGzh-HrEoFDwZnCJq27bVp5atdMT9o") + essr, _ = exchanging.exchange(route='/essr/req', sender=hab.pre, diger=diger, + modifiers=dict(src=hab.pre, dest=recHab.pre)) + ims = hab.endorse(serder=essr, pipelined=False) + ims.extend(core.Counter(core.Codens.ESSRPayloadGroup, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) + ims.extend(texter[0].qb64b) + + parsing.Parser().parse(ims=ims, kvy=recHby.kvy, exc=exc) + assert recHby.db.exns.get(keys=(essr.said,)) is None + + def test_exchanger(): + with habbing.openHab(name="sid", base="test", salt=b'0123456789abcdef') as (hby, hab), \ + habbing.openHab(name="rec", base="test", salt=b'0123456789abcdef') as (recHby, recHab): + exc = exchanging.Exchanger(hby=recHby, handlers=[]) + + msg = hab.makeOwnInception() + recHab.psr.parseOne(ims=msg) + + ser, sigs, _ = hab.getOwnEvent(sn=0) + + sadsig = signing.SadPathSigGroup(pather=coring.Pather(path=[]), sigers=sigs) + act = bytearray() + pather = coring.Pather(path=["e"]) + sadsig.transpose(pather) + act.extend(sadsig.proof) + + # create the forward message with payload embedded at `a` field + fwd, _ = exchanging.exchange(route='/fwd', sender=hab.pre, + modifiers=dict(pre="EBCAFG", topic="/delegation"), + payload={}, embeds=dict(evt=ser.raw)) + with pytest.raises(kering.MissingSignatureError): + exc.processEvent(serder=fwd, source=hab.kever.prefixer, tsgs=None) + + assert recHby.db.epse.get(keys=(fwd.said,)) is not None + exc.processEscrowPartialSigned() + assert recHby.db.epse.get(keys=(fwd.said,)) is not None + + # Set the PSE timeout artifically low to trigger removal + exc.TimeoutPSE = 0.00001 + exc.processEscrowPartialSigned() + assert recHby.db.epse.get(keys=(fwd.said,)) is None + + +def test_exchange_ps_escrow_timeout(): with habbing.openHab(name="sid", base="test", salt=b'0123456789abcdef') as (hby, hab), \ habbing.openHab(name="rec", base="test", salt=b'0123456789abcdef') as (recHby, recHab): mbx = storing.Mailboxer(hby=hby) @@ -76,7 +165,7 @@ def test_exchanger(): def test_hab_exchange(mockHelpingNowUTC): - with habbing.openHby() as hby: + with habbing.openHby(salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: hab = hby.makeHab(name="test") assert hab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" @@ -98,24 +187,26 @@ def test_hab_exchange(mockHelpingNowUTC): data = dict(m="Let's create a registry") msg = hab.exchange(route="/multisig/registry/incept", recipient="", payload=data, embeds=embeds) - assert msg == (b'{"v":"KERI10JSON0003a0_","t":"exn","d":"ELkHqph-Tj4LGHYfFfoVmJJo09S2gp6ci8rK96upIAKE",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"","p":"",' - b'"dt":"2021-01-01T00:00:00.000000+00:00","r":"/multisig/registry/incept","q":{},"a":{"i":"",' - b'"m":"Let\'s create a registry"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp",' - b'"d":"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB",' - b'"i":"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB",' - b'"ii":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":"0","c":[],"bt":"0","b":[],' - b'"n":"AH3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10JSON000138_",' - b'"t":"ixn","d":"EFuFnevyDFfpWG6il-6Qcv0ne0ZIItLwanCwI-SU8A9j",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":"1",' - b'"p":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3",' - b'"a":[{"i":"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","s":0,' - b'"d":"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB"}]},' - b'"d":"EL5Nkm6T7HG_0GW6uwqYSZwlH23khtXvsVE-dq8eO_eE"}}-FABEIaGMMWJFPmtXznY1IIiKDIrg' - b'-vIyge6mBl2QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3' - b'-AABAAB-teJc_7zot5TAZT6lQi2-GlBzMHXICvt3tIYoPo2gYXF7PpWDozo3y3wVW9mgHln-1DvQlqn9Aip1YnBgKUQB' - b'-LAa5AACAA-e-ixn-AABAADprTWp4llIzVzBM7VVsDOgXVJdoiVXutsWJEbDJ2pMdjXjNi1xKALBSZ1ZgRoUsD' - b'--LgUQkXIdjLoQ19XPvJMJ') + assert msg == (b'{"v":"KERI10JSON0003a0_","t":"exn","d":"ELkHqph-Tj4LGHYfFfoVmJJo' + b'09S2gp6ci8rK96upIAKE","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2Q' + b'V8dDjI3","rp":"","p":"","dt":"2021-01-01T00:00:00.000000+00:00",' + b'"r":"/multisig/registry/incept","q":{},"a":{"i":"","m":"Let\'s cr' + b'eate a registry"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp",' + b'"d":"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","i":"EI6hBlgk' + b'WoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","ii":"EIaGMMWJFPmtXznY1IIi' + b'KDIrg-vIyge6mBl2QV8dDjI3","s":"0","c":[],"bt":"0","b":[],"n":"AH' + b'3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10J' + b'SON000138_","t":"ixn","d":"EFuFnevyDFfpWG6il-6Qcv0ne0ZIItLwanCwI' + b'-SU8A9j","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":' + b'"1","p":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","a":[{"i"' + b':"EI6hBlgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB","s":0,"d":"EI6hB' + b'lgkWoJgkZyfLW35_UyM4nIK44OgsSwFR_WOfvVB"}]},"d":"EL5Nkm6T7HG_0GW' + b'6uwqYSZwlH23khtXvsVE-dq8eO_eE"}}-FABEIaGMMWJFPmtXznY1IIiKDIrg-vI' + b'yge6mBl2QV8dDjI30AAAAAAAAAAAAAAAAAAAAAAAEIaGMMWJFPmtXznY1IIiKDIr' + b'g-vIyge6mBl2QV8dDjI3-AABAAB-teJc_7zot5TAZT6lQi2-GlBzMHXICvt3tIYo' + b'Po2gYXF7PpWDozo3y3wVW9mgHln-1DvQlqn9Aip1YnBgKUQB-LAa5AACAA-e-ixn' + b'-AABAADprTWp4llIzVzBM7VVsDOgXVJdoiVXutsWJEbDJ2pMdjXjNi1xKALBSZ1Z' + b'gRoUsD--LgUQkXIdjLoQ19XPvJMJ') exn = serdering.SerderKERI(raw=msg) @@ -138,25 +229,27 @@ def test_hab_exchange(mockHelpingNowUTC): data = dict(m="Lets create this registry instead") msg = hab2.exchange(route="/multisig/registry/incept", payload=data, recipient="", dig=exn.said, embeds=embeds) - assert msg == (b'{"v":"KERI10JSON0003d6_","t":"exn","d":"EPO_XC9nwSixqSoOvsHymFr-l3udclHBdOh4OUEqZ33P",' - b'"i":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","rp":"",' - b'"p":"ELkHqph-Tj4LGHYfFfoVmJJo09S2gp6ci8rK96upIAKE","dt":"2021-01-01T00:00:00.000000+00:00",' - b'"r":"/multisig/registry/incept","q":{},"a":{"i":"","m":"Lets create this registry instead"},' - b'"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp",' - b'"d":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB",' - b'"i":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB",' - b'"ii":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"0","c":[],"bt":"0","b":[],' - b'"n":"AH3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10JSON000138_",' - b'"t":"ixn","d":"EOek9JVKNeuW-5UNeHYCTDe70_GtvRwP672oWMNBJpA5",' - b'"i":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"1",' - b'"p":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2",' - b'"a":[{"i":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","s":0,' - b'"d":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB"}]},' - b'"d":"EM3gLTzQ9GmKd50Rlm_kiIkeYkxb004eoOsWahz70TqJ' - b'"}}-FABEIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc20AAAAAAAAAAAAAAAAAAAAAAAEIREQlatUJODbKogZ' - b'fa3IqXZ90XdZA0qJMVliI61Bcc2-AABAADY5nUsBgL23ulcrTgkV09hSzktNHZSlEH1zmVpEggrGgQUq0tLQeOXztUFD' - b'xNQ4Kq2ddIYDVz6d_y0kkU3__YJ-LAa5AACAA-e-ixn-AABAACaoxfQp5L_Gd0nKqJXMbLTXzkrJJDd8RFxWdTSesAMy' - b'dUzmJQlGt0T9h8L7SwIrq8yBinj990PLJHl7sXmq04I') + assert msg == (b'{"v":"KERI10JSON0003d6_","t":"exn","d":"EPO_XC9nwSixqSoOvsHymFr-' + b'l3udclHBdOh4OUEqZ33P","i":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVli' + b'I61Bcc2","rp":"","p":"ELkHqph-Tj4LGHYfFfoVmJJo09S2gp6ci8rK96upIA' + b'KE","dt":"2021-01-01T00:00:00.000000+00:00","r":"/multisig/regis' + b'try/incept","q":{},"a":{"i":"","m":"Lets create this registry in' + b'stead"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp","d":"EB5mt' + b's6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","i":"EB5mts6qrWOZrxjma6' + b'lSTjAdPZ0NSHM1HC3IndbS_giB","ii":"EIREQlatUJODbKogZfa3IqXZ90XdZA' + b'0qJMVliI61Bcc2","s":"0","c":[],"bt":"0","b":[],"n":"AH3-1EZWXU9I' + b'0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"ixn":{"v":"KERI10JSON000138_' + b'","t":"ixn","d":"EOek9JVKNeuW-5UNeHYCTDe70_GtvRwP672oWMNBJpA5","' + b'i":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"1","p":"E' + b'IREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","a":[{"i":"EB5mts6q' + b'rWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","s":0,"d":"EB5mts6qrWOZrxj' + b'ma6lSTjAdPZ0NSHM1HC3IndbS_giB"}]},"d":"EM3gLTzQ9GmKd50Rlm_kiIkeY' + b'kxb004eoOsWahz70TqJ"}}-FABEIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI' + b'61Bcc20AAAAAAAAAAAAAAAAAAAAAAAEIREQlatUJODbKogZfa3IqXZ90XdZA0qJM' + b'VliI61Bcc2-AABAADY5nUsBgL23ulcrTgkV09hSzktNHZSlEH1zmVpEggrGgQUq0' + b'tLQeOXztUFDxNQ4Kq2ddIYDVz6d_y0kkU3__YJ-LAa5AACAA-e-ixn-AABAACaox' + b'fQp5L_Gd0nKqJXMbLTXzkrJJDd8RFxWdTSesAMydUzmJQlGt0T9h8L7SwIrq8yBi' + b'nj990PLJHl7sXmq04I') # Test exn from non-transferable AID hab = hby.makeHab(name="test1", transferable=False) @@ -167,16 +260,18 @@ def test_hab_exchange(mockHelpingNowUTC): ) msg = hab.exchange(route="/multisig/registry/incept", payload=data, embeds=embeds, recipient="") - assert msg == (b'{"v":"KERI10JSON00026b_","t":"exn","d":"EMBm0p7fCIqJrP4Z-PBI-yEvXin_-eY1dU4XTCM9ykRC",' - b'"i":"BJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6-tFVlCeiZPG","rp":"","p":"",' - b'"dt":"2021-01-01T00:00:00.000000+00:00","r":"/multisig/registry/incept","q":{},"a":{"i":"",' - b'"m":"Lets create this registry instead"},"e":{"vcp":{"v":"KERI10JSON00010f_","t":"vcp",' - b'"d":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB",' - b'"i":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB",' - b'"ii":"EIREQlatUJODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"0","c":[],"bt":"0","b":[],' - b'"n":"AH3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},' - b'"d":"ENC6w8wUj-Gp_RpAJN5q4Lf00IHstzNLUvkh3ZvgHGP_"}}-CABBJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6' - b'-tFVlCeiZPG0BB-sQs0WS9wsyuT4hXQD7rbczSfpnQz21wZGYucRkE0ynKy5draELEKBsckeD0Im1i' - b'-kIfMEdbY08YqVfSrEoAA-LAl5AACAA-e-vcp-CABBJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6' - b'-tFVlCeiZPG0BDjOC4j0Co6P0giMylR47149eJ8Yf_hO' - b'-32_TpY77KMVCWCf0U8GuZPIN76R2zsyT_eARvS_zQsX1ebjl3PMP0D') + assert msg == (b'{"v":"KERI10JSON00026b_","t":"exn","d":"EMBm0p7fCIqJrP4Z-PBI-yEv' + b'Xin_-eY1dU4XTCM9ykRC","i":"BJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6-tFV' + b'lCeiZPG","rp":"","p":"","dt":"2021-01-01T00:00:00.000000+00:00",' + b'"r":"/multisig/registry/incept","q":{},"a":{"i":"","m":"Lets cre' + b'ate this registry instead"},"e":{"vcp":{"v":"KERI10JSON00010f_",' + b'"t":"vcp","d":"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","i"' + b':"EB5mts6qrWOZrxjma6lSTjAdPZ0NSHM1HC3IndbS_giB","ii":"EIREQlatUJ' + b'ODbKogZfa3IqXZ90XdZA0qJMVliI61Bcc2","s":"0","c":[],"bt":"0","b":' + b'[],"n":"AH3-1EZWXU9I0fv3Iz_9ZIhjj13JO7u4GNFYC3-l8_K-"},"d":"ENC6' + b'w8wUj-Gp_RpAJN5q4Lf00IHstzNLUvkh3ZvgHGP_"}}-CABBJZ_LF61JTCCSCIw2' + b'Q4ozE2MsbRC4m-N6-tFVlCeiZPG0BB-sQs0WS9wsyuT4hXQD7rbczSfpnQz21wZG' + b'YucRkE0ynKy5draELEKBsckeD0Im1i-kIfMEdbY08YqVfSrEoAA-LAl5AACAA-e-' + b'vcp-CABBJZ_LF61JTCCSCIw2Q4ozE2MsbRC4m-N6-tFVlCeiZPG0BDjOC4j0Co6P' + b'0giMylR47149eJ8Yf_hO-32_TpY77KMVCWCf0U8GuZPIN76R2zsyT_eARvS_zQsX' + b'1ebjl3PMP0D') diff --git a/tests/test_kering.py b/tests/test_kering.py new file mode 100644 index 000000000..a8f2941ff --- /dev/null +++ b/tests/test_kering.py @@ -0,0 +1,922 @@ +# -*- encoding: utf-8 -*- +""" +tests.test_kering module + +""" +import re +import json +from dataclasses import asdict, astuple + +import cbor2 as cbor +import msgpack + +import pytest + +from keri import kering +from keri.kering import Protocolage, Protocols +from keri.kering import Kindage, Kinds +from keri.kering import Ilkage, Ilks +from keri.kering import ColdCodex, ColdDex, TraitCodex, TraitDex +from keri.kering import (Versionage, Version, MAXVERFULLSPAN, + versify, deversify, Rever, Smellage, smell, + VER1FULLSPAN, VER1TERM, VEREX1, + VER2FULLSPAN, VER2TERM, VEREX2, + VEREX) + + +from keri.kering import VersionError, ProtocolError, KindError +from keri.help.helping import (intToB64, intToB64b, b64ToInt, B64_CHARS, + codeB64ToB2, codeB2ToB64, Reb64, nabSextets) + + +def test_protocols(): + """ + Test protocols namedtuple instance Protocols + """ + + assert isinstance(Protocols, Protocolage) + + assert Protocols.keri == 'KERI' + assert Protocols.acdc == 'ACDC' + + assert 'KERI' in Protocols + assert 'ACDC' in Protocols + + """End Test""" + + + +def test_version_regex(): + """ + Test version string regexing + + VER1FULLSPAN = 17 # number of characters in full version string + VER1TERM = b'_' + VEREX1 = b'(?P[A-Z]{4})(?P[0-9a-f])(?P[0-9a-f])(?P[A-Z]{4})(?P[0-9a-f]{6})_' + + VER2FULLSPAN = 16 # number of characters in full version string + VER2TERM = b'.' + VEREX2 = b'(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[A-Z]{4})(?P[0-9A-Za-z_-]{4}).' + + VEREX = VEREX2 + b'|' + VEREX1 + """ + + + # Test VEREX2 by itself + pattern = re.compile(VEREX2) # compile is faster + + vs = b'KERICAAJSONAAAB.' + + match = pattern.match(vs) + assert match + + full = match.group() # not group args so returns full match + assert full == vs + span = len(full) + assert span == VER2FULLSPAN + assert VER2TERM == chr(full[-1]).encode("utf-8") + assert ord(VER2TERM) == full[-1] + + groups = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + + assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') + + # Test VEREX with combined VEREXes + pattern = re.compile(VEREX) # compile is faster + + vs = b'KERICAAJSONAAAB.' + + match = pattern.match(vs) + assert match + + full = match.group() # not group args so returns full match + assert full == vs + span = len(full) + assert span == VER2FULLSPAN + assert VER2TERM == chr(full[-1]).encode("utf-8") + assert ord(VER2TERM) == full[-1] + + groups = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + + assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') + + vs = b'KERI10JSON000002_' + + match = pattern.match(vs) + assert match + + full = match.group() # not group args so returns full match + assert full == vs + span = len(full) + assert span == VER1FULLSPAN + assert VER1TERM == chr(full[-1]).encode("utf-8") + assert ord(VER1TERM) == full[-1] + + groups = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + + assert groups == (b'KERI', b'1', b'0', b'JSON', b'000002') + + raw = b'{"vs":"KERICAAJSONAAAB.","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"}' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + span = len(full) + assert span == VER2FULLSPAN + assert VER2TERM == chr(full[-1]).encode("utf-8") + assert ord(VER2TERM) == full[-1] + + groups = match.group("proto2", + "major2", + "minor2", + "kind2", + "size2") + + assert groups == (b'KERI', b'C', b'AA', b'JSON', b'AAAB') + + + raw = b'{"vs":"KERI10JSON000002_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"}' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + span = len(full) + assert span == VER1FULLSPAN + assert VER1TERM == chr(full[-1]).encode("utf-8") + assert ord(VER1TERM) == full[-1] + + groups = match.group("proto1", + "major1", + "minor1", + "kind1", + "size1") + + assert groups == (b'KERI', b'1', b'0', b'JSON', b'000002') + + """End Test""" + + +def test_smell(): + """ + Test smell function to parse into Serializations + """ + + raw = b'{"vs":"KERICAAJSONAAAB.","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + assert smell(raw) == Smellage(proto='KERI', + vrsn=Versionage(major=2, minor=0), + kind='JSON', + size=1, + gvrsn=None) + + raw = b'{"vs":"KERI10JSON000002_","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + assert smell(raw) == Smellage(proto='KERI', + vrsn=Versionage(major=1, minor=0), + kind='JSON', + size=2, + gvrsn=None) + + raw = b'{"vs":"KERICAAJSONAAABX.","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + with pytest.raises(ProtocolError): + smell(raw) + + raw = b'{"vs":"KERI1XJSON000002_","t":"ixn","d":"EPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf"}' + with pytest.raises(VersionError): + smell(raw) + + """End Test""" + + +def test_snuff(): + """ + Test snuff for looking ahead at CESR native messages from stream + + VER0FULLSPAN = 12 # number of characters in full version string + VEREX0 = b'0N(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})' + + + """ + + # version field in CESR native serialization + VFFULLSPAN = 12 # number of characters in full version string + VFREX = b'0N(?P[A-Z]{4})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})(?P[0-9A-Za-z_-])(?P[0-9A-Za-z_-]{2})' + + Revfer = re.compile(VFREX) # compile is faster + + MAXVFOFFSET = 12 + + SNUFFSIZE = MAXVFOFFSET + VFFULLSPAN + + def snatch(match, size=0): + """ Returns: + smellage (Smellage): named tuple extracted from version string regex match + (protocol, version, kind, size) + + Parameters: + match (re.Match): instance of Match class + size (int): provided size to substitute when missing + + Notes: + regular expressions work with memoryview objects not just bytes or + bytearrays + """ + full = match.group() # full matched version string + if len(full) == VFFULLSPAN: + proto, major, minor, gmajor, gminor = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + proto = proto.decode("utf-8") + if proto not in Protocols: + raise ProtocolError(f"Invalid protocol type = {proto}.") + vrsn = Versionage(major=b64ToInt(major), minor=b64ToInt(minor)) + if vrsn.major < 2: # version2 vs but major < 2 + raise VersionError(f"Incompatible {vrsn=} with version string.") + + gvrsn = Versionage(major=b64ToInt(gmajor), minor=b64ToInt(gminor)) + if gvrsn.major < 2: # version2 vs but major < 2 + raise VersionError(f"Incompatible {gvrsn=} with CESR native version" + f"field.") + kind = Kinds.cesr + size = size + else: + raise VersionError(f"Bad snatch.") + + return Smellage(proto=proto, vrsn=vrsn, kind=kind, size=size, gvrsn=gvrsn) + + + def snuff(raw, size=0): + """Extract and return instance of Smellage from version string inside + raw serialization. + + Returns: + smellage (Smellage): named Tuple of (protocol, version, kind, size) + + Parameters: + raw (bytearray) of serialized incoming message stream. Assumes start + of stream is JSON, CBOR, or MGPK field map with first field + is labeled 'v' and value is version string. + size (int): provided size to substitute when missing + + """ + if len(raw) < SNUFFSIZE: + raise kering.ShortageError(f"Need more raw bytes to smell full version string.") + + match = Rever.search(raw) # Rever regex takes bytes/bytearray not str + if not match or match.start() > MAXVFOFFSET: + raise kering.VersionError(f"Invalid version string from smelled raw = " + f"{raw[: SNUFFSIZE]}.") + + return snatch(match, size=size) + + + + + + + + #pattern = re.compile(VFREX) # compile is faster + pattern = Revfer + + vv = b'0NKERICAACAB' + + match = pattern.match(vv) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-FAM' + vv + assert raw == b'-FAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-0FAAAAM' + vv + assert raw == b'-0FAAAAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + + vv = b'0NKERICAACAB' + + match = pattern.match(vv) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-FAM' + vv + assert raw == b'-FAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + raw = b'-0FAAAAM' + vv + assert raw == b'-0FAAAAM0NKERICAACAB' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'C', b'AA', b'C', b'AB') + + vv = b'0NKERI______' + raw = b'-0FAAAAM' + vv + assert raw == b'-0FAAAAM0NKERI______' + + match = pattern.search(raw) + assert match + + full = match.group() # not group args so returns full match + assert full == vv + span = len(full) + assert span == VFFULLSPAN + + groups = match.group("proto0", + "major0", + "minor0", + "gmajor0", + "gminor0") + + assert groups == (b'KERI', b'_', b'__', b'_', b'__') + + + + #raw = b'-FAM0NKERICAACABXicpEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #assert smell(raw) == Smellage(protocol='KERI', + #version=Versionage(major=2, minor=0), + #kind='CESR', + #size=0) + + #raw =b'-0FAAAAM0NKERICAACABXrotEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #assert smell(raw, size=4096) == Smellage(protocol='KERI', + #version=Versionage(major=2, minor=0), + #kind='CESR', + #size=4096) + + #raw =b'-0FAAAAM0MKERICAACABXrotEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #with pytest.raises(VersionError): + #smell(raw) + + #raw =b'-0FAAAAMNKERICAACABXrotEPTgL0UEOa8xUWBqghryJYMLOd2eYjmclndQN4bArjSf' + #with pytest.raises(VersionError): + #smell(raw) + + + +def test_serials(): + """ + Test Serializations namedtuple instance Serials + """ + + assert Version == Versionage(major=1, minor=0) + + assert isinstance(Kinds, Kindage) + + assert Kinds.json == 'JSON' + assert Kinds.mgpk == 'MGPK' + assert Kinds.cbor == 'CBOR' + assert Kinds.cesr == 'CESR' + + assert 'JSON' in Kinds + assert 'MGPK' in Kinds + assert 'CBOR' in Kinds + assert 'CESR' in Kinds + + Vstrings = Kindage(json=versify(kind=Kinds.json, size=0), + mgpk=versify(kind=Kinds.mgpk, size=0), + cbor=versify(kind=Kinds.cbor, size=0), + cesr=versify(kind=Kinds.cesr, size=0)) + + + assert Vstrings.json == 'KERI10JSON000000_' + assert Vstrings.mgpk == 'KERI10MGPK000000_' + assert Vstrings.cbor == 'KERI10CBOR000000_' + assert Vstrings.cesr == 'KERI10CESR000000_' + + icp = dict(vs=Vstrings.json, + pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', + sn='0001', + ilk='icp', + dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', + sith=1, + keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], + nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', + toad=0, + wits=[], + cnfg=[], + ) + + rot = dict(vs=Vstrings.json, + pre='AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM', + sn='0001', + ilk='rot', + dig='DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfS', + sith=1, + keys=['AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM'], + nxt='DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM', + toad=0, + cuts=[], + adds=[], + data=[], + ) + + icps = json.dumps(icp, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + assert len(icps) == 303 + assert icps == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'","sn":"0001","ilk":"icp","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' + b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"wits":[],"cnfg":[]}') + + match = Rever.search(icps) + assert match.group() == Vstrings.json.encode("utf-8") + + rots = json.dumps(rot, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + assert len(rots) == 313 + assert rots == (b'{"vs":"KERI10JSON000000_","pre":"AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'","sn":"0001","ilk":"rot","dig":"DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'S","sith":1,"keys":["AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM"],"nxt":"' + b'DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5CM","toad":0,"cuts":[],"adds":[],"' + b'data":[]}') + + match = Rever.search(rots) + assert match.group() == Vstrings.json.encode("utf-8") + + icp["vs"] = Vstrings.mgpk + icps = msgpack.dumps(icp) + assert len(icps) == 264 + assert icps == (b'\x8b\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3icp\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' + b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' + b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' + b'CM\xa4toad\x00\xa4wits\x90\xa4cnfg\x90') + + match = Rever.search(icps) + assert match.group() == Vstrings.mgpk.encode("utf-8") + + rot["vs"] = Vstrings.mgpk + rots = msgpack.dumps(rot) + assert len(rots) == 270 + assert rots == (b'\x8c\xa2vs\xb1KERI10MGPK000000_\xa3pre\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAf' + b'SVPzhzS6b5CM\xa2sn\xa40001\xa3ilk\xa3rot\xa3dig\xd9,DVPzhzS6b5CMaU6JR2nmwy' + b'Z-i0d8JZAoTNZH3ULvYAfS\xa4sith\x01\xa4keys\x91\xd9,AaU6JR2nmwyZ-i0d8JZAoTNZ' + b'H3ULvYAfSVPzhzS6b5CM\xa3nxt\xd9,DZ-i0d8JZAoTNZH3ULvaU6JR2nmwyYAfSVPzhzS6b5' + b'CM\xa4toad\x00\xa4cuts\x90\xa4adds\x90\xa4data\x90') + + match = Rever.search(rots) + assert match.group() == Vstrings.mgpk.encode("utf-8") + + icp["vs"] = Vstrings.cbor + icps = cbor.dumps(icp) + assert len(icps) == 264 + assert icps == (b'\xabbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'bsnd0001cilkcicpcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' + b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' + b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dwits\x80dcnfg\x80') + + match = Rever.search(icps) + assert match.group() == Vstrings.cbor.encode("utf-8") + + rot["vs"] = Vstrings.cbor + rots = cbor.dumps(rot) + assert len(rots) == 270 + assert rots == (b'\xacbvsqKERI10CBOR000000_cprex,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CM' + b'bsnd0001cilkcrotcdigx,DVPzhzS6b5CMaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSdsith\x01' + b'dkeys\x81x,AaU6JR2nmwyZ-i0d8JZAoTNZH3ULvYAfSVPzhzS6b5CMcnxtx,DZ-i0d8JZAoTNZ' + b'H3ULvaU6JR2nmwyYAfSVPzhzS6b5CMdtoad\x00dcuts\x80dadds\x80ddata\x80') + + match = Rever.search(rots) + assert match.group() == Vstrings.cbor.encode("utf-8") + """Done Test""" + + +def test_versify_v1(): + """ + Test Versify support + """ + + assert VER1FULLSPAN == MAXVERFULLSPAN + + # default version is version 1 + + vs = versify() # defaults + assert vs == "KERI10JSON000000_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.json + assert vrsn == Version + assert size == 0 + + vs = versify(kind=Kinds.json, size=65) + assert vs == "KERI10JSON000041_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.json + assert vrsn == Version + assert size == 65 + + vs = versify(protocol=Protocols.acdc, kind=Kinds.json, size=86) + assert vs == "ACDC10JSON000056_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.acdc + assert kind == Kinds.json + assert vrsn == Version + assert size == 86 + + vs = versify(kind=Kinds.mgpk, size=0) + assert vs == "KERI10MGPK000000_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.mgpk + assert vrsn == Version + assert size == 0 + + vs = versify(kind=Kinds.mgpk, size=65) + assert vs == "KERI10MGPK000041_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.mgpk + assert vrsn == Version + assert size == 65 + + vs = versify(kind=Kinds.cbor, size=0) + assert vs == "KERI10CBOR000000_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.cbor + assert vrsn == Version + assert size == 0 + + vs = versify(kind=Kinds.cbor, size=65) + assert vs == "KERI10CBOR000041_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.cbor + assert vrsn == Version + assert size == 65 + + vs = versify(version=Versionage(major=1, minor=1)) # defaults + assert vs == "KERI11JSON000000_" + assert len(vs) == VER1FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.json + assert vrsn == (1, 1) + assert size == 0 + + # test bad version strings + vs = "KERI20JSON000000_" + with pytest.raises(VersionError): + smellage = deversify(vs) + + vs = "ABLE10JSON000000_" + with pytest.raises(ProtocolError): + smellage = deversify(vs) + + vs = "KERI10MSON000000_" + with pytest.raises(KindError): + smellage = deversify(vs) + + + """End Test""" + + +def test_versify_v2(): + """ + Test Versify support + """ + version = Versionage(major=2, minor=0) + assert version == (2, 0) + + vs = versify(version=version) # defaults + assert vs == "KERICAAJSONAAAA." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.json + assert vrsn == version + assert size == 0 + + vs = versify(version=version, kind=Kinds.json, size=65) + assert vs == "KERICAAJSONAABB." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.json + assert vrsn == version + assert size == 65 + + vs = versify(protocol=Protocols.acdc, version=version, kind=Kinds.json, size=86) + assert vs == "ACDCCAAJSONAABW." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.acdc + assert kind == Kinds.json + assert version == version + assert size == 86 + + vs = versify(version=version, kind=Kinds.mgpk, size=0) + assert vs == 'KERICAAMGPKAAAA.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.mgpk + assert vrsn == version + assert size == 0 + + vs = versify(version=version, kind=Kinds.mgpk, size=65) + assert vs == 'KERICAAMGPKAABB.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.mgpk + assert vrsn == version + assert size == 65 + + vs = versify(version=version, kind=Kinds.cbor, size=0) + assert vs == 'KERICAACBORAAAA.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.cbor + assert vrsn == version + assert size == 0 + + vs = versify(version=version, kind=Kinds.cbor, size=65) + assert vs == 'KERICAACBORAABB.' + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.cbor + assert vrsn == version + assert size == 65 + + vs = versify(version=Versionage(major=2, minor=1)) # defaults + assert vs == "KERICABJSONAAAA." + assert len(vs) == VER2FULLSPAN + proto, vrsn, kind, size, opt = deversify(vs) + assert proto == Protocols.keri + assert kind == Kinds.json + assert vrsn == (2, 1) + assert size == 0 + + # test bad version strings + vs = "KERIBAAJSONAAAA." + with pytest.raises(VersionError): + smellage = deversify(vs) + + vs = "KERI20JSON000000" + with pytest.raises(VersionError): + smellage = deversify(vs) + + vs = "ABLECAAJSONAAAA." + with pytest.raises(ProtocolError): + smellage = deversify(vs) + + vs = "KERICAAMSONAAAA." + with pytest.raises(KindError): + smellage = deversify(vs) + + """End Test""" + + +def test_colddex(): + """ + Test ColdDex instance of ColdCodex dataclass + """ + + assert isinstance(ColdDex, ColdCodex) + + assert asdict(ColdDex) == \ + { + 'AnB64': 0, + 'CtB64': 1, + 'OpB64': 2, + 'JSON': 3, + 'MGPK1': 4, + 'CBOR': 5, + 'MGPK2': 6, + 'CtOpB2': 7 + } + + assert 0o0 in ColdDex + assert 0o1 in ColdDex + assert 0o2 in ColdDex + assert 0o3 in ColdDex + assert 0o4 in ColdDex + assert 0o5 in ColdDex + assert 0o6 in ColdDex + assert 0o7 in ColdDex + + """End Test""" + + +def test_ilks(): + """ + Test Ilkage namedtuple instance Ilks + """ + Ilks = Ilkage(icp='icp', rot='rot', ixn='ixn', dip='dip', drt='drt', + rct='rct', + qry='qry', rpy='rpy', xip='xip', exn='exn', pro='pro', bar='bar', + vcp='vcp', vrt='vrt', iss='iss', rev='rev', bis='bis', brv='brv', + rip='rip', upd='upd', acd='acd', ace='ace', + sch='sch', att='att', agg='agg', edg='edg', rul='rul') + + assert isinstance(Ilks, Ilkage) + + for fld in Ilks._fields: + assert fld == getattr(Ilks, fld) + + assert 'icp' in Ilks + assert Ilks.icp == 'icp' + assert 'rot' in Ilks + assert Ilks.rot == 'rot' + assert 'ixn' in Ilks + assert Ilks.ixn == 'ixn' + assert 'dip' in Ilks + assert Ilks.dip == 'dip' + assert 'drt' in Ilks + assert Ilks.drt == 'drt' + assert 'rct' in Ilks + assert Ilks.rct == 'rct' + assert 'qry' in Ilks + assert Ilks.qry == 'qry' + assert 'rpy' in Ilks + assert Ilks.rpy == 'rpy' + assert 'exn' in Ilks + assert Ilks.exn == 'exn' + + + assert 'pro' in Ilks + assert Ilks.pro == 'pro' + assert 'bar' in Ilks + assert Ilks.bar == 'bar' + + + assert 'vcp' in Ilks + assert Ilks.vcp == 'vcp' + assert 'vrt' in Ilks + assert Ilks.vrt == 'vrt' + assert 'iss' in Ilks + assert Ilks.iss == 'iss' + assert 'rev' in Ilks + assert Ilks.rev == 'rev' + assert 'bis' in Ilks + assert Ilks.bis == 'bis' + assert 'brv' in Ilks + assert Ilks.brv == 'brv' + + assert 'rip' in Ilks + assert Ilks.rip == 'rip' + assert 'upd' in Ilks + assert Ilks.upd == 'upd' + + assert 'acd' in Ilks + assert Ilks.acd == 'acd' + assert 'ace' in Ilks + assert Ilks.ace == 'ace' + assert 'sch' in Ilks + assert Ilks.sch == 'sch' + assert 'att' in Ilks + assert Ilks.att == 'att' + assert 'agg' in Ilks + assert Ilks.agg == 'agg' + assert 'edg' in Ilks + assert Ilks.edg == 'edg' + assert 'rul' in Ilks + assert Ilks.rul == 'rul' + + """End Test """ + +def test_traitdex(): + """ + Test TraitDex instance of TraitCodex dataclass + """ + + assert isinstance(TraitDex, TraitCodex) + + assert asdict(TraitDex) == \ + { + 'EstOnly': 'EO', + 'DoNotDelegate': 'DND', + 'RegistrarBackers': 'RB', + 'NoBackers': 'NB', + 'NoRegistrarBackers': 'NRB', + 'DelegateIsDelegator': 'DID', + } + + assert 'EO' in TraitDex + assert 'DND' in TraitDex + assert 'RB' in TraitDex + assert 'NB' in TraitDex + assert 'NRB' in TraitDex + assert 'DID' in TraitDex + + """End Test""" + + + +if __name__ == "__main__": + test_protocols() + + test_version_regex() + test_smell() + test_snuff() + test_serials() + test_versify_v1() + test_versify_v2() + test_ilks() + test_colddex() + test_traitdex() diff --git a/tests/vc/test_protocoling.py b/tests/vc/test_protocoling.py index cff1ae6d0..69a5b4853 100644 --- a/tests/vc/test_protocoling.py +++ b/tests/vc/test_protocoling.py @@ -4,35 +4,39 @@ """ -from keri.app import habbing, notifying +from keri import core, kering from keri.core import coring, scheming, parsing from keri.core.eventing import SealEvent + from keri.peer import exchanging from keri.vc import protocoling from keri.vc.proving import credential from keri.vdr import credentialing, verifying +from keri.app import habbing, notifying def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingNowUTC): """ Test IPEX exchange protocol """ - sidSalt = coring.Salter(raw=b'0123456789abcdef').qb64 - assert sidSalt == '0AAwMTIzNDU2Nzg5YWJjZGVm' - wanSalt = coring.Salter(raw=b'wann-the-witness').qb64 - assert wanSalt == '0AB3YW5uLXRoZS13aXRuZXNz' + sidSalt = core.Salter(raw=b'0123456789abcdef').qb64 + assert sidSalt == '0AAUiJMii_rPXXCiLTEEaDT7' + wanSalt = core.Salter(raw=b'wann-the-witness').qb64 + assert wanSalt == '0AAUiJMii_rPXXCiLTEEaDT7' - with (habbing.openHby(name="red", base="test") as redHby, + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + + with (habbing.openHby(name="red", base="test", salt=default_salt) as redHby, habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby): seeder.seedSchema(redHby.db) seeder.seedSchema(sidHby.db) sidHab = sidHby.makeHab(name="test") sidPre = sidHab.pre - assert sidPre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + assert sidPre == "EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl" redHab = redHby.makeHab(name="test") redPre = redHab.pre - assert redPre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + assert redPre == "EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl" sidRgy = credentialing.Regery(hby=sidHby, name="bob", temp=True) sidVer = verifying.Verifier(hby=sidHby, reger=sidRgy.reger) @@ -67,12 +71,12 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN data=d, status=issuer.regk) - assert creder.said == "EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG" + assert creder.said == "EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH" iss = issuer.issue(said=creder.said) - assert iss.raw == (b'{"v":"KERI10JSON0000ed_","t":"iss","d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afq' - b'dCIZjMy3","i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0","ri":"E' - b'O0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","dt":"2021-06-27T21:26:21.23325' + assert iss.raw == (b'{"v":"KERI10JSON0000ed_","t":"iss","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-i' + b'LLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","ri":"E' + b'B-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","dt":"2021-06-27T21:26:21.23325' b'7+00:00"}') rseal = SealEvent(iss.pre, "0", iss.said)._asdict() sidHab.interact(data=[rseal]) @@ -84,27 +88,28 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN sidRgy.processEscrows() msg = creder.raw - assert msg == (b'{"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","ri":"EO0_SyqPS1-EVYSITak' - b'YpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' - b'","a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-27T2' - b'1:26:21.233257+00:00","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LE' + assert msg == (b'{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH",' + b'"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJo' + b'AVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' + b'","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T2' + b'1:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LE' b'I":"254900OPPU84GM83MG36"}}') + atc = bytearray(msg) - atc.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + atc.extend(core.Counter(core.Codens.SealSourceTriples, count=1, gvrsn=kering.Vrsn_1_0).qb64b) atc.extend(coring.Prefixer(qb64=iss.pre).qb64b) atc.extend(coring.Seqner(sn=0).qb64b) atc.extend(iss.saidb) - assert atc == (b'{"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qw' - b'vHdlvgfOyG","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","' - b'ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCn' - b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EF2__B6DiLQHpdJZ' - b'_C0bddxy2o6nXIHEwchO9yylr3xx","dt":"2021-06-27T21:26:21.233257+0' - b'0:00","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"' - b'254900OPPU84GM83MG36"}}-IABEDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHd' - b'lvgfOyG0AAAAAAAAAAAAAAAAAAAAAAAEK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9' - b'afqdCIZjMy3') + assert atc == (b'{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103' + b'V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","' + b'ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8F' + b'w1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+0' + b'0:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"' + b'254900OPPU84GM83MG36"}}-IABEElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-' + b'4E-ClXH0AAAAAAAAAAAAAAAAAAAAAAAECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H' + b'8-iLLAmIUPO') parsing.Parser().parseOne(ims=bytes(atc), vry=sidVer) # Successfully parsed credential is now saved in database. @@ -115,29 +120,26 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN apply0, apply0atc = protocoling.ipexApplyExn(sidHab, message="Please give me a credential", schema=schema, recp=redPre, attrs={}) - assert apply0.raw == (b'{"v":"KERI10JSON000175_","t":"exn","d":"EK9Q7n1y-Rlf-A7-T0uWSKuOlHZHBy_4zVaNp60GxXEr",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"","p":"",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/apply","q":{},"a":{"m":"Please ' - b'give me a credential","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{},' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3"},"e":{}}') + assert apply0.raw == (b'{"v":"KERI10JSON000175_","t":"exn","d":"EHVK5cO32UQJCkpK9RqRP_ONViK8u3JNXn73' + b'nJ8hdmXr","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"",' + b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/apply","q":{},"a":{"m":"P' + b'lease give me a credential","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1k' + b'C","a":{},"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{}}') # No requirements for apply, except that its first, no `p` assert ipexhan.verify(serder=apply0) is True offer0, offer0atc = protocoling.ipexOfferExn(sidHab, "How about this", acdc=creder.raw, apply=apply0) - assert offer0.raw == (b'{"v":"KERI10JSON0002f8_","t":"exn","d":"ED-uXbt7hRH3cmQY9vtwmcPOGvdmPEq_bnQ4sgQK9KhB",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"",' - b'"p":"EK9Q7n1y-Rlf-A7-T0uWSKuOlHZHBy_4zVaNp60GxXEr",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/offer","q":{},"a":{"m":"How about ' - b'this"},"e":{"acdc":{"v":"ACDC10JSON000197_",' - b'"d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3",' - b'"ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4",' - b'"s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC",' - b'"a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx",' - b'"dt":"2021-06-27T21:26:21.233257+00:00",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"254900OPPU84GM83MG36"}},' - b'"d":"EOVRKHUAEjvfyWzQ8IL4icBiaVuy_CSTse_W_AssaAeE"}}') + assert offer0.raw == (b'{"v":"KERI10JSON0002f8_","t":"exn","d":"ENdVOCsP5Xz57qs1xa_msznozvBs6Ii0_JRo' + b'i6tp2NBu","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EH' + b'VK5cO32UQJCkpK9RqRP_ONViK8u3JNXn73nJ8hdmXr","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/offer","q":{},"a":{"m":"How about this"},"e":{"acdc":{"v"' + b':"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":' + b'"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv' + b'5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a' + b'":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26' + b':21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"' + b'254900OPPU84GM83MG36"}},"d":"EOG-KWyllXlb2HVIuewN1YJAOT304PaSczyt3V5Z878S"}}') # This should fail because it is not first and the apply isn't persisted yet assert ipexhan.verify(serder=offer0) is False @@ -164,29 +166,26 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Let's see if we can spurn a message we previously accepted. spurn0, spurn0atc = protocoling.ipexSpurnExn(sidHab, "I reject you", spurned=apply0) - assert spurn0.raw == (b'{"v":"KERI10JSON000125_","t":"exn","d":"EP9mGfCLrVs-A2KiSwruQ8bdQVlPIiIFZ2t4f6Dj4gnc",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"",' - b'"p":"EK9Q7n1y-Rlf-A7-T0uWSKuOlHZHBy_4zVaNp60GxXEr",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/spurn","q":{},"a":{"m":"I reject ' - b'you"},"e":{}}') + assert spurn0.raw == (b'{"v":"KERI10JSON000125_","t":"exn","d":"EHijfrof83z7JeFR-wJO9Ptgl-PieQHhKC-F' + b'bZIDvGvM","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EH' + b'VK5cO32UQJCkpK9RqRP_ONViK8u3JNXn73nJ8hdmXr","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') # This will fail, we've already responded with an offer assert ipexhan.verify(spurn0) is False # Now lets try an offer without a pointer back to a reply offer1, offer1atc = protocoling.ipexOfferExn(sidHab, "Here a credential offer", acdc=creder.raw) - assert offer1.raw == (b'{"v":"KERI10JSON0002d5_","t":"exn","d":"EJC9_GH0TeYJ3_cyutkS1gZfcgaGUQnk3v7F_gwJShVM",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"","p":"",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/offer","q":{},"a":{"m":"Here a ' - b'credential offer"},"e":{"acdc":{"v":"ACDC10JSON000197_",' - b'"d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3",' - b'"ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4",' - b'"s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC",' - b'"a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx",' - b'"dt":"2021-06-27T21:26:21.233257+00:00",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"254900OPPU84GM83MG36"}},' - b'"d":"EOVRKHUAEjvfyWzQ8IL4icBiaVuy_CSTse_W_AssaAeE"}}') + assert offer1.raw == (b'{"v":"KERI10JSON0002d5_","t":"exn","d":"EC8fiu3IoCex-7uhTskkEodJOiQYQpO61l3Y' + b'HCXWuuFi","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"",' + b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/offer","q":{},"a":{"m":"H' + b'ere a credential offer"},"e":{"acdc":{"v":"ACDC10JSON000197_","d":"EElymNmgs' + b'1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xy' + b'a8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcC' + b'nVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOs' + b'KKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00","i":"EMl4RhuR_Jxp' + b'iMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83MG36"}},"d":"EOG-KW' + b'yllXlb2HVIuewN1YJAOT304PaSczyt3V5Z878S"}}') # Will work because it is starting a new conversation assert ipexhan.verify(serder=offer1) is True @@ -198,11 +197,11 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN assert serder.ked == offer1.ked agree, argeeAtc = protocoling.ipexAgreeExn(sidHab, "I'll accept that offer", offer=offer0) - assert agree.raw == (b'{"v":"KERI10JSON00012f_","t":"exn","d":"ELLFpKUv8qt6UKaNFj2_s-3Hs1vFeRgWdq_LIQm2HEER",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"",' - b'"p":"ED-uXbt7hRH3cmQY9vtwmcPOGvdmPEq_bnQ4sgQK9KhB",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/agree","q":{},"a":{"m":"I\'ll ' - b'accept that offer"},"e":{}}') + assert agree.raw == (b'{"v":"KERI10JSON00012f_","t":"exn","d":"ECU3UjnSY1_6Wl3aYEW19jaGiKuyFh_chIQQ' + b'w48bcT_X","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EN' + b'dVOCsP5Xz57qs1xa_msznozvBs6Ii0_JRoi6tp2NBu","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/agree","q":{},"a":{"m":"I\'ll accept that offer"},"e":' + b'{}}') # Can not create an agree without an offer, so this will pass since it has an offer that has no response assert ipexhan.verify(serder=agree) is True @@ -217,28 +216,24 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN anc = sidHab.makeOwnEvent(sn=2) grant0, grant0atc = protocoling.ipexGrantExn(sidHab, message="Here's a credential", recp=sidHab.pre, acdc=msg, iss=iss.raw, anc=anc) - assert grant0.raw == (b'{"v":"KERI10JSON000539_","t":"exn","d":"EBXaaGfKREvo3UbvNEtgTiZySHe71tTU4-ZmcFETVdn8",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"","p":"",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/grant","q":{},"a":{"m":"Here\'s a ' - b'credential","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3"},"e":{"acdc":{' - b'"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3",' - b'"ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4",' - b'"s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC",' - b'"a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx",' - b'"dt":"2021-06-27T21:26:21.233257+00:00",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"254900OPPU84GM83MG36"}},' - b'"iss":{"v":"KERI10JSON0000ed_","t":"iss",' - b'"d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afqdCIZjMy3",' - b'"i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0",' - b'"ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4",' - b'"dt":"2021-06-27T21:26:21.233257+00:00"},"anc":{"v":"KERI10JSON00013a_","t":"ixn",' - b'"d":"EOjAxp-AMLzicGz2h-DxvMK9kicajpZEwdN8-8k54hvz",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":"2",' - b'"p":"EGKglEgIpdHuhuwl-IiSDG9x094gMrRxVaXGgXvCzCYM",' - b'"a":[{"i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0",' - b'"d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afqdCIZjMy3"}]},' - b'"d":"EI5mZXZ84Su4DrEUOxtl-NaUURQtTJeAn12xf146beg3"}}') + assert grant0.raw == (b'{"v":"KERI10JSON000539_","t":"exn","d":"ELnjKvzdgO57JZwG3giIScoOeTB0rLuevniv' + b'zRE5DbTE","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"",' + b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/grant","q":{},"a":{"m":"H' + b'ere\'s a credential","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},' + b'"e":{"acdc":{"v":"ACDC10JSON000197_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103' + b'V3-4E-ClXH","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VA' + b'F7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvaf' + b'X3bHQ9Gkk1kC","a":{"d":"EO9_6NattzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"' + b'2021-06-27T21:26:21.233257+00:00","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8A' + b'N-tiUbl","LEI":"254900OPPU84GM83MG36"}},"iss":{"v":"KERI10JSON0000ed_","t":"' + b'iss","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO","i":"EElymNmgs1u0mSa' + b'oCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXe' + b'w8Yo6Z3w9mQUQ","dt":"2021-06-27T21:26:21.233257+00:00"},"anc":{"v":"KERI10JS' + b'ON00013a_","t":"ixn","d":"EGhSHKIV5-nkeirdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":' + b'"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","s":"2","p":"ED1kkh5_ECYriK-j' + b'2gSv6Zjr5way88XVhwRCxk5zoTRG","a":[{"i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V' + b'3-4E-ClXH","s":"0","d":"ECUw7AdWEE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d"' + b':"EJ4-dlS9ktlb9HDWPYc0IJ2hS2NbvnCQBhUsFSkEPwIo"}}') assert ipexhan.verify(serder=grant0) is True @@ -251,11 +246,10 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Let's see if we can spurn a message we previously accepted. spurn1, spurn1atc = protocoling.ipexSpurnExn(sidHab, "I reject you", spurned=grant0) - assert spurn1.raw == (b'{"v":"KERI10JSON000125_","t":"exn","d":"EIoMDwEvyR4j43W5Hh9CyJ4ttwNHlrmABwyUvKHhF9mp",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"",' - b'"p":"EBXaaGfKREvo3UbvNEtgTiZySHe71tTU4-ZmcFETVdn8",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/spurn","q":{},"a":{"m":"I reject ' - b'you"},"e":{}}') + assert spurn1.raw == (b'{"v":"KERI10JSON000125_","t":"exn","d":"ELHFilyCgYvVq6vczgPQc7ZuRMs7Cv10U_h-' + b'2LIvJ1-Z","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EL' + b'njKvzdgO57JZwG3giIScoOeTB0rLuevnivzRE5DbTE","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/spurn","q":{},"a":{"m":"I reject you"},"e":{}}') smsg = bytearray(spurn1.raw) smsg.extend(spurn1atc) parsing.Parser().parse(ims=smsg, exc=sidExc) @@ -265,29 +259,25 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # Now we'll run a grant pointing back to the agree all the way to the database grant1, grant1atc = protocoling.ipexGrantExn(sidHab, message="Here's a credential", acdc=msg, iss=iss.raw, recp=sidHab.pre, anc=anc, agree=agree) - assert grant1.raw == (b'{"v":"KERI10JSON000565_","t":"exn","d":"ENy1ktZHowD73mn0vJL-xpTzCDpa4RuISZldAZImiKD_",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"",' - b'"p":"ELLFpKUv8qt6UKaNFj2_s-3Hs1vFeRgWdq_LIQm2HEER",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/grant","q":{},"a":{"m":"Here\'s a ' - b'credential","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3"},"e":{"acdc":{' - b'"v":"ACDC10JSON000197_","d":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3",' - b'"ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4",' - b'"s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC",' - b'"a":{"d":"EF2__B6DiLQHpdJZ_C0bddxy2o6nXIHEwchO9yylr3xx",' - b'"dt":"2021-06-27T21:26:21.233257+00:00",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","LEI":"254900OPPU84GM83MG36"}},' - b'"iss":{"v":"KERI10JSON0000ed_","t":"iss",' - b'"d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afqdCIZjMy3",' - b'"i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0",' - b'"ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4",' - b'"dt":"2021-06-27T21:26:21.233257+00:00"},"anc":{"v":"KERI10JSON00013a_","t":"ixn",' - b'"d":"EOjAxp-AMLzicGz2h-DxvMK9kicajpZEwdN8-8k54hvz",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","s":"2",' - b'"p":"EGKglEgIpdHuhuwl-IiSDG9x094gMrRxVaXGgXvCzCYM",' - b'"a":[{"i":"EDkftEwWBpohjTpemh_6xkaGNuoDsRU3qwvHdlvgfOyG","s":"0",' - b'"d":"EK2WxcpF3oL1yqS3Z8i08WDYkHDcYhJL9afqdCIZjMy3"}]},' - b'"d":"EI5mZXZ84Su4DrEUOxtl-NaUURQtTJeAn12xf146beg3"}}') + assert grant1.raw == (b'{"v":"KERI10JSON000565_","t":"exn","d":"EHAY_L6Ig4k_5qIw6uH-QZwBswWLfwVzCqaG' + b'sjtdnubK","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EC' + b'U3UjnSY1_6Wl3aYEW19jaGiKuyFh_chIQQw48bcT_X","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/grant","q":{},"a":{"m":"Here\'s a credential","i":"EMl' + b'4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl"},"e":{"acdc":{"v":"ACDC10JSON0001' + b'97_","d":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","i":"EMl4RhuR_JxpiMd' + b'1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w' + b'9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EO9_6Nat' + b'tzsFiO8Fw1cxjYmDjOsKKSbootn-wXn9S3iB","dt":"2021-06-27T21:26:21.233257+00:00' + b'","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","LEI":"254900OPPU84GM83' + b'MG36"}},"iss":{"v":"KERI10JSON0000ed_","t":"iss","d":"ECUw7AdWEE3fvr7dgbFDXj' + b'0CEZuJTTa_H8-iLLAmIUPO","i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","' + b's":"0","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","dt":"2021-06-27T' + b'21:26:21.233257+00:00"},"anc":{"v":"KERI10JSON00013a_","t":"ixn","d":"EGhSHK' + b'IV5-nkeirdkqzqsvmeF1FXw_yH8NvPSAY1Rgyd","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn' + b'9Xya8AN-tiUbl","s":"2","p":"ED1kkh5_ECYriK-j2gSv6Zjr5way88XVhwRCxk5zoTRG","a' + b'":[{"i":"EElymNmgs1u0mSaoCeOtSsNOROLuqOz103V3-4E-ClXH","s":"0","d":"ECUw7AdW' + b'EE3fvr7dgbFDXj0CEZuJTTa_H8-iLLAmIUPO"}]},"d":"EJ4-dlS9ktlb9HDWPYc0IJ2hS2Nbvn' + b'CQBhUsFSkEPwIo"}}') assert ipexhan.verify(serder=grant1) is True gmsg = bytearray(grant1.raw) @@ -298,11 +288,11 @@ def test_ipex(seeder, mockCoringRandomNonce, mockHelpingNowIso8601, mockHelpingN # And now the last... admit the granted credential to complete the full flow admit0, admit0atc = protocoling.ipexAdmitExn(sidHab, "Thanks for the credential", grant=grant1) - assert admit0.raw == (b'{"v":"KERI10JSON000132_","t":"exn","d":"EAnQEaL-jSGK22VSbPN7WAUWVcxJ9LV8S5fORVAqVQzN",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","rp":"",' - b'"p":"ENy1ktZHowD73mn0vJL-xpTzCDpa4RuISZldAZImiKD_",' - b'"dt":"2021-06-27T21:26:21.233257+00:00","r":"/ipex/admit","q":{},"a":{"m":"Thanks for ' - b'the credential"},"e":{}}') + assert admit0.raw == (b'{"v":"KERI10JSON000132_","t":"exn","d":"EMxU5rfeqKnZzrbqnL7weXQGaC8Zum4qowj2' + b'eiL6GxqL","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","rp":"","p":"EH' + b'AY_L6Ig4k_5qIw6uH-QZwBswWLfwVzCqaGsjtdnubK","dt":"2021-06-27T21:26:21.233257' + b'+00:00","r":"/ipex/admit","q":{},"a":{"m":"Thanks for the credential"},"e":{' + b'}}') assert ipexhan.verify(serder=admit0) is True amsg = bytearray(admit0.raw) diff --git a/tests/vc/test_proving.py b/tests/vc/test_proving.py index 128209153..383e38e5b 100644 --- a/tests/vc/test_proving.py +++ b/tests/vc/test_proving.py @@ -5,11 +5,17 @@ """ import pytest -from keri.app import habbing +from keri import kering +from keri.kering import Versionage, Vrsn_1_0 + +from keri import core from keri.core import coring, scheming, parsing, serdering -from keri.core.coring import Serials, Counter, CtrDex, Prefixer, Seqner, Diger, Siger +from keri.core import counting, Counter, Codens +from keri.core.coring import Kinds, Prefixer, Seqner, Diger +from keri.core.indexing import Siger from keri.core.scheming import CacheResolver -from keri.kering import Versionage + +from keri.app import habbing from keri.vc.proving import credential from keri.vdr import verifying, credentialing @@ -17,7 +23,7 @@ def test_proving(mockHelpingNowIso8601): """Test credential proof with SerderACDC""" - sidSalt = coring.Salter(raw=b'0123456789abcdef').qb64 + sidSalt = core.Salter(raw=b'0123456789abcdef').qb64 with habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby: sidHab = sidHby.makeHab(name="test", ) @@ -69,15 +75,15 @@ def test_proving(mockHelpingNowIso8601): creder = serdering.SerderACDC(raw=msg) # Creder(raw=msg) proof = msg[creder.size:] - ctr = Counter(qb64b=proof, strip=True) - assert ctr.code == CtrDex.AttachedMaterialQuadlets + ctr = Counter(qb64b=proof, strip=True, gvrsn=Vrsn_1_0) + assert ctr.code == counting.CtrDex_1_0.AttachmentGroup assert ctr.count == 52 pags = ctr.count * 4 assert len(proof) == pags - ctr = Counter(qb64b=proof, strip=True) - assert ctr.code == CtrDex.TransIdxSigGroups + ctr = Counter(qb64b=proof, strip=True, gvrsn=Vrsn_1_0) + assert ctr.code == counting.CtrDex_1_0.TransIdxSigGroups assert ctr.count == 1 prefixer = Prefixer(qb64b=proof, strip=True) @@ -89,8 +95,8 @@ def test_proving(mockHelpingNowIso8601): diger = Diger(qb64b=proof, strip=True) assert diger.qb64 == sidHab.kever.serder.said - ictr = Counter(qb64b=proof, strip=True) - assert ictr.code == CtrDex.ControllerIdxSigs + ictr = Counter(qb64b=proof, strip=True, gvrsn=Vrsn_1_0) + assert ictr.code == counting.CtrDex_1_0.ControllerIdxSigs isigers = [] for i in range(ictr.count): @@ -108,12 +114,12 @@ def test_proving(mockHelpingNowIso8601): def test_credentialer(): """Test SerderACDC as credential""" - with pytest.raises(ValueError): + with pytest.raises(kering.InvalidValueError): serdering.SerderACDC() # Creder() sub = dict(a=123, b="abc", issuanceDate="2021-06-27T21:26:21.233257+00:00") d = dict( - v=coring.versify(proto=coring.Protos.acdc, kind=Serials.json, size=0), + v=coring.versify(protocol=coring.Protocols.acdc, kind=Kinds.json, size=0), d="", i="EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi", s="abc", @@ -125,7 +131,7 @@ def test_credentialer(): creder = serdering.SerderACDC(sad=d) # Creder(ked=d) assert creder.said == said - assert creder.kind == Serials.json + assert creder.kind == Kinds.json assert creder.issuer == "EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi" assert creder.schema == "abc" assert creder.attrib == sub @@ -136,21 +142,24 @@ def test_credentialer(): b'"i":"EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi","s":"abc","a":{"a":123,"b' b'":"abc","issuanceDate":"2021-06-27T21:26:21.233257+00:00"}}') - raw1, ked1, knd1, ver1, knd1, size1 = creder._exhale(sad=d) - assert raw1 == creder.raw - assert knd1 == Serials.json - assert ked1 == d + raw1 = creder.raw + ver1 = creder.vrsn + knd1 = creder.kind + sad1 = creder.sad + + assert knd1 == Kinds.json + assert sad1 == d assert ver1 == Versionage(major=1, minor=0) creder = serdering.SerderACDC(raw=raw1) # Creder(raw=raw1) - assert creder.kind == Serials.json + assert creder.kind == Kinds.json assert creder.issuer == "EF6maPM_d5ZN7U3NRFC1-6TM7k_E00_a8AG9YyLA4uWi" assert creder.sad == d assert creder.size == 211 d2 = dict(d) d2['d'] = "" - d2["v"] = coring.versify(proto=coring.Protos.acdc, kind=Serials.cbor, size=0) + d2["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Kinds.cbor, size=0) _, d2 = coring.Saider.saidify(sad=d2) creder = serdering.SerderACDC(sad=d2) # Creder(ked=d2) @@ -176,7 +185,7 @@ def test_credentialer(): assert creder.sad == d2 d3 = dict(d) - d3["v"] = coring.versify(proto=coring.Protos.acdc, kind=Serials.mgpk, size=0) + d3["v"] = coring.versify(protocol=coring.Protocols.acdc, kind=Kinds.mgpk, size=0) _, d3 = coring.Saider.saidify(sad=d3) creder = serdering.SerderACDC(sad=d3) # Creder(ked=d3) @@ -244,26 +253,25 @@ def test_privacy_preserving_credential(mockHelpingNowIso8601): engagementContextRole="Project Manager", ) - salt = coring.Salter(raw=b'0123456789abcdef').qb64 cred = credential(schema="EZllThM1rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q", recipient="EM_S2MdMaKgP6P2Yyno6-flV6GqrwPencTIw8tCMR7iB", private=True, - private_credential_nonce=coring.Salter(raw=b'0123456789abcdef').qb64, - private_subject_nonce=coring.Salter(raw=b'abcdef0123456789').qb64, + private_credential_nonce=core.Salter(raw=b'0123456789abcdef').qb64, + private_subject_nonce=core.Salter(raw=b'abcdef0123456789').qb64, issuer="EMZeK1yLZd1JV6Ktdq_YUt-YbyoTWB9UMcFzuiDly2Y6", data=d, status="ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M") assert cred.size == len(cred.raw) assert "u" in cred.sad - print(cred.raw) + assert cred.raw == (b'{"v":"ACDC10JSON00021c_","d":"EMMDzhHHlpQP0XNMRThDeIFkYD1WkDHF7Tp-8kt8X5pn",' - b'"u":"0AAwMTIzNDU2Nzg5YWJjZGVm","i":"EMZeK1yLZd1JV6Ktdq_YUt-YbyoTWB9UMcFzuiDl' - b'y2Y6","ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M","s":"EZllThM1rLBSM' - b'Z_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q","a":{"d":"EK3MRnlg-bMUnHtYKyZ8HD_IbBeI0v4N8' - b'YB4UnNVBqrv","u":"0ABhYmNkZWYwMTIzNDU2Nzg5","i":"EM_S2MdMaKgP6P2Yyno6-flV6Gq' - b'rwPencTIw8tCMR7iB","dt":"2021-06-27T21:26:21.233257+00:00","LEI":"254900OPPU' - b'84GM83MG36","personLegalName":"John Doe","engagementContextRole":"Project Ma' - b'nager"}}') + b'"u":"0AAwMTIzNDU2Nzg5YWJjZGVm","i":"EMZeK1yLZd1JV6Ktdq_YUt-YbyoTWB9UMcFzuiDly2Y6",' + b'"ri":"ETQoH02zJRCTNz-Wl3nnkUD_RVSzSwcoNvmfa18AWt3M",' + b'"s":"EZllThM1rLBSMZ_ozM1uAnFvSfC0N1jaQ42aKU5sCZ5Q",' + b'"a":{"d":"EK3MRnlg-bMUnHtYKyZ8HD_IbBeI0v4N8YB4UnNVBqrv","u":"0ABhYmNkZWYwMTIzNDU2Nzg5",' + b'"i":"EM_S2MdMaKgP6P2Yyno6-flV6GqrwPencTIw8tCMR7iB","dt":"2021-06-27T21:26:21.233257+00:00",' + b'"LEI":"254900OPPU84GM83MG36","personLegalName":"John Doe","engagementContextRole":"Project ' + b'Manager"}}') """End Test""" @@ -285,7 +293,7 @@ def test_credential_parsator(): status=issuer.regk) msg = bytearray(creder.raw) - msg.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + msg.extend(Counter(Codens.SealSourceTriples, count=1, gvrsn=Vrsn_1_0).qb64b) msg.extend(hab.kever.prefixer.qb64b) msg.extend(coring.Seqner(sn=hab.kever.sn).qb64b) msg.extend(hab.kever.serder.said.encode("utf-8")) diff --git a/tests/vc/test_walleting.py b/tests/vc/test_walleting.py index 2f1c55f23..ddbbf4e35 100644 --- a/tests/vc/test_walleting.py +++ b/tests/vc/test_walleting.py @@ -3,21 +3,23 @@ tests.vc.walleting module """ - -from keri.app import habbing, signing +from keri import core, kering from keri.core import coring, parsing from keri.core.eventing import SealEvent + +from keri.app import habbing + from keri.vc.proving import credential from keri.vdr import verifying, credentialing def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): - sidSalt = coring.Salter(raw=b'0123456789abcdef').qb64 + sidSalt = core.Salter(raw=b'0123456789abcdef').qb64 with habbing.openHby(name="sid", base="test", salt=sidSalt) as sidHby: sidHab = sidHby.makeHab(name="test") seeder.seedSchema(db=sidHby.db) - assert sidHab.pre == "EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3" + assert sidHab.pre == "EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl" schema = "EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC" credSubject = dict( @@ -41,7 +43,7 @@ def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): schema=schema, data=credSubject, status=issuer.regk) - assert creder.said == "EOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMhar4R5a3OfB" + assert creder.said == "EAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFpZj2oPTc7vM" iss = issuer.issue(said=creder.said) rseal = SealEvent(iss.pre, "0", iss.said)._asdict() @@ -54,26 +56,27 @@ def test_wallet(seeder, mockCoringRandomNonce, mockHelpingNowIso8601): sidReg.processEscrows() msg = bytearray(creder.raw) - msg.extend(coring.Counter(coring.CtrDex.SealSourceTriples, count=1).qb64b) + msg.extend(core.Counter(core.Codens.SealSourceTriples, count=1, + gvrsn=kering.Vrsn_1_0).qb64b) msg.extend(coring.Prefixer(qb64=iss.pre).qb64b) msg.extend(coring.Seqner(sn=0).qb64b) msg.extend(iss.saidb) - assert msg == (b'{"v":"ACDC10JSON000197_","d":"EOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMh' - b'ar4R5a3OfB","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","' - b'ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCn' - b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EFKsAdq9CZF_w9yv' - b'ia8RiRdDeXLMjR6q7Lp7FKKIgJx-","i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIy' - b'ge6mBl2QV8dDjI3","dt":"2021-06-27T21:26:21.233257+00:00","LEI":"' - b'254900OPPU84GM83MG36"}}-IABEOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMhar4' - b'R5a3OfB0AAAAAAAAAAAAAAAAAAAAAAAEIMoFDXHR3cNF0fADC5nLPme34n-ZsMEu' - b'n6eDFvN8Jgc') + assert msg == (b'{"v":"ACDC10JSON000197_","d":"EAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFp' + b'Zj2oPTc7vM","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","' + b'ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCn' + b'VRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC","a":{"d":"EMyHBc5ZujkNFm9t' + b'nrwBU2nmp_qodcV4aDW28pwbDdgb","i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ov' + b'vn9Xya8AN-tiUbl","dt":"2021-06-27T21:26:21.233257+00:00","LEI":"' + b'254900OPPU84GM83MG36"}}-IABEAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFpZj2' + b'oPTc7vM0AAAAAAAAAAAAAAAAAAAAAAAEAOV3Ie3yAvGU1MwbIHr816qewYzRLlvX' + b'NreKmXtJShe') - ser = (b'{"v":"ACDC10JSON000197_","d":"EOavcpdGvk4sTXjOQiNxHeNf3HYMjMINMhar4R5a3OfB",' - b'"i":"EIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3","ri":"EO0_SyqPS1-EVYSITak' - b'YpUHaUZZpZGsjaXFOaO_kCfS4","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' - b'","a":{"d":"EFKsAdq9CZF_w9yvia8RiRdDeXLMjR6q7Lp7FKKIgJx-","i":"EIaGMMWJFPmtX' - b'znY1IIiKDIrg-vIyge6mBl2QV8dDjI3","dt":"2021-06-27T21:26:21.233257+00:00","LE' + ser = (b'{"v":"ACDC10JSON000197_","d":"EAP1MTFwoSZ7P9Ym9yIqBvihjqZYpilpFpZj2oPTc7vM",' + b'"i":"EMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","ri":"EB-u4VAF7A7_GR8PXJo' + b'AVHv5X9vjtXew8Yo6Z3w9mQUQ","s":"EMQWEcCnVRk1hatTNyK3sIykYSrrFvafX3bHQ9Gkk1kC' + b'","a":{"d":"EMyHBc5ZujkNFm9tnrwBU2nmp_qodcV4aDW28pwbDdgb","i":"EMl4RhuR_Jxpi' + b'Md1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl","dt":"2021-06-27T21:26:21.233257+00:00","LE' b'I":"254900OPPU84GM83MG36"}}') parsing.Parser().parse(ims=msg, vry=verifier) diff --git a/tests/vdr/__init__.py b/tests/vdr/__init__.py index ffbf7bcb8..fc59a8734 100644 --- a/tests/vdr/__init__.py +++ b/tests/vdr/__init__.py @@ -2,7 +2,10 @@ """ Test utilities for vdr """ -from keri.core import coring + +from keri import core + +from keri import app from keri.app import habbing def buildHab(db, ks, name="test"): @@ -15,7 +18,7 @@ def buildHab(db, ks, name="test"): # create secrecies secrecies = [[signer.qb64] for signer in - coring.Salter(raw=raw).signers(count=8, + core.Salter(raw=raw).signers(count=8, path="name", temp=True)] diff --git a/tests/vdr/test_eventing.py b/tests/vdr/test_eventing.py index 3da4f055c..7a0f4618f 100644 --- a/tests/vdr/test_eventing.py +++ b/tests/vdr/test_eventing.py @@ -6,9 +6,10 @@ import pytest from keri.app import habbing, keeping +from keri.core import Signer from keri.core import coring, serdering from keri.core import eventing as keventing -from keri.core.coring import versify, Serials, Ilks, MtrDex, Prefixer, Signer, Seqner, Saider +from keri.core.coring import versify, Kinds, Ilks, MtrDex, Prefixer, Seqner, Saider from keri.db import basing from keri.db.dbing import snKey, dgKey from keri.kering import Version, EmptyMaterialError, DerivationError, MissingAnchorError, ValidationError, \ @@ -31,17 +32,17 @@ def test_incept(mockCoringRandomNonce): # no backers, allowed to add later serder = eventing.incept(pre, baks=[], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00010f_","t":"vcp","d":"ELkr1d1qLyIXVPfuaEjkDJIfgxXarUjoB0RN' - b'mswxHnvD","i":"ELkr1d1qLyIXVPfuaEjkDJIfgxXarUjoB0RNmswxHnvD","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"0","b":[],"n":"A9X' - b'fpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON0000fb_","t":"vcp","d":"EE7_RC-GkGy_N_IbOjFHLhqDekFNeSxygvIZ' + b'JBf1SPbA","i":"EE7_RC-GkGy_N_IbOjFHLhqDekFNeSxygvIZJBf1SPbA","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"0","b":[],"n":"0AA' + b'UiJMii_rPXXCiLTEEaDT7"}') # no backers allowed serder = eventing.incept(pre, baks=[], cnfg=[keventing.TraitDex.NoBackers], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON000113_","t":"vcp","d":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPX' - b'Bsg2LvxA","i":"EBoBPh3N5nr1tItAUCkXNx3vShB_Be6iiQPXBsg2LvxA","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":["NB"],"bt":"0","b":[],"n":' - b'"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON0000ff_","t":"vcp","d":"EM2g-TbMWL98wHMiY7fs1eBRE0uVNL21xnwa' + b'ys_i9K1Q","i":"EM2g-TbMWL98wHMiY7fs1eBRE0uVNL21xnways_i9K1Q","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":["NB"],"bt":"0","b":[],"n":' + b'"0AAUiJMii_rPXXCiLTEEaDT7"}') # no backers allows, one attempted with pytest.raises(ValueError): @@ -67,45 +68,43 @@ def test_incept(mockCoringRandomNonce): serder = eventing.incept(pre, baks=[bak1], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I' - b'6AwmpZ6x","i":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I6AwmpZ6x","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000129_","t":"vcp","d":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg' + b'8uYupLCq","i":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg8uYupLCq","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') # 3 backers serder = eventing.incept(pre, baks=[bak1, bak2, bak3], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00019b_","t":"vcp","d":"ECw3lpxt56pRMeh7VnZStljriimMfm5or0Ka' - b'aNv7PSHl","i":"ECw3lpxt56pRMeh7VnZStljriimMfm5or0KaaNv7PSHl","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"3","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' - b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"A9XfpxIl1LcIkMh' - b'USCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000187_","t":"vcp","d":"EI7cIxSfVdi8WhU3dj-laWIkNI1a-7MS6F-5' + b'Ss758cv9","i":"EI7cIxSfVdi8WhU3dj-laWIkNI1a-7MS6F-5Ss758cv9","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"3","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' + b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"0AAUiJMii_rPXXC' + b'iLTEEaDT7"}') # one backer, with threshold serder = eventing.incept(pre, toad=1, baks=[bak1], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I' - b'6AwmpZ6x","i":"EHYybz8zckAN_4rj8JfuKMogmWr7gRLKxf_I6AwmpZ6x","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000129_","t":"vcp","d":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg' + b'8uYupLCq","i":"EP_pM8RMJ2K25lepbhe5vYJnxYf4OrebU_eg8uYupLCq","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"1","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') # 3 backers, with threshold serder = eventing.incept(pre, toad=2, baks=[bak1, bak2, bak3], code=MtrDex.Blake3_256) - assert serder.raw == (b'{"v":"KERI10JSON00019b_","t":"vcp","d":"EGqiHHLDIMr7VYNhDSnmsjOMpaLUOBRAgxvb' - b'rXYSFDfk","i":"EGqiHHLDIMr7VYNhDSnmsjOMpaLUOBRAgxvbrXYSFDfk","ii":"DAtNTPnDF' - b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"2","b":["EAvR3p8V9' - b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' - b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"A9XfpxIl1LcIkMh' - b'USCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert serder.raw == (b'{"v":"KERI10JSON000187_","t":"vcp","d":"EDnhbkM1_hVVC_df54giqQ0qq6722QCXM9BE' + b'AK-GFqiF","i":"EDnhbkM1_hVVC_df54giqQ0qq6722QCXM9BEAK-GFqiF","ii":"DAtNTPnDF' + b'BnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM","s":"0","c":[],"bt":"2","b":["EAvR3p8V9' + b'5W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc","DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBA' + b'vN9XNU","DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw"],"n":"0AAUiJMii_rPXXC' + b'iLTEEaDT7"}') """ End Test """ @@ -321,97 +320,7 @@ def test_backer_issue_revoke(mockHelpingNowUTC): """ End Test """ -def test_prefixer(): - pre = "DAtNTPnDFBnmlO6J44LXCrzZTAmpe-82b7BmQGtL4QhM" - vs = versify(version=Version, kind=Serials.json, size=0) - - with pytest.raises(EmptyMaterialError): - prefixer = Prefixer() - - # vcp, backers allowed no backers - # ["v", "d", "i", "s", "t", "bt", "b", "c"] - ked = dict(v=vs, - d="", # qb64 SAID - i="", # qb64 pre - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=0, - b=[], - c=[], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EDj0Kq4tFBNGKxpdfX2nCfIvYJ-v1MJ24H1dsPfUqzmB' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - # Invalid event type - #["v", "i", "s", "t", "ri", "dt"] - ked = dict(v=vs, - d="", # qb64 SAID - i="", - s="{:x}".format(0), - t=Ilks.iss, - ri="", - dt="", - ) - #with pytest.raises(DerivationError): - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - - # vcp, no backers allowed - ked = dict(v=vs, - d="", # qb64 SAID - i="", - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=0, - b=[], - c=[keventing.TraitDex.NoBackers], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EDz0QmMxf4Dk0C9uiP-y3okN-Bej2IAXSj8UwQgb3NsL' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - bak1 = "EAvR3p8V95W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc" - bak2 = "DBEpNJeSJjxo6oAxkNE8eCOJg2HRPstqkeHWBAvN9XNU" - bak3 = "DCxo-P4W_Z0xXTfoA3_4DMPn7oi0mLCElOWJDpC0nQXw" - - # vcp, one backer - ked = dict(v=vs, - d="", # qb64 SAID - i="", - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=1, - b=[bak1], - c=[], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EIDRsRwJQNw2ujTeoztpEPxN6XRmBB2bnxWlOzJ9OQHk' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - # vcp, many backers - ked = dict(v=vs, - d="", # qb64 SAID - i="", - ii=pre, - s="{:x}".format(0), - t=Ilks.vcp, - bt=2, - b=[bak1, bak2, bak3], - c=[], - ) - prefixer = Prefixer(ked=ked, code=MtrDex.Blake3_256) - assert prefixer.qb64 == 'EMsbtrSOa_lNwDMhpdXphmbItPBcaB0qSboopPo9Ub-s' - assert prefixer.verify(ked=ked) is True - assert prefixer.verify(ked=ked, prefixed=True) is False - - """ End Test """ def test_tever_escrow(mockCoringRandomNonce): @@ -427,7 +336,7 @@ def test_tever_escrow(mockCoringRandomNonce): cnfg=[], code=MtrDex.Blake3_256) regk = vcp.pre - assert regk == 'EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTcS59ssXg5' + assert regk == 'EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7BwNUl6wms' assert vcp.said == vcp.pre assert vcp.ked["ii"] == hab.pre @@ -456,13 +365,13 @@ def test_tever_escrow(mockCoringRandomNonce): dgkey = dgKey(pre=regk, dig=vcp.said) vcp = reg.getTvt(dgkey) - assert bytes(vcp) == (b'{"v":"KERI10JSON00010f_","t":"vcp","d":"EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTc' - b'S59ssXg5","i":"EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTcS59ssXg5","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"0","b":[],"n":"A9X' - b'fpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert bytes(vcp) == (b'{"v":"KERI10JSON0000fb_","t":"vcp","d":"EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7B' + b'wNUl6wms","i":"EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7BwNUl6wms","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"0","b":[],"n":"0AA' + b'UiJMii_rPXXCiLTEEaDT7"}') dig = reg.getTae(snKey(pre=regk, sn=0)) - assert bytes(dig) == b'EJfe0JaRiSWTCnasczAPkvZ-M2dd5nnhxcTcS59ssXg5' + assert bytes(dig) == b'EEu4cX0EqO9mTqsNgxDgCT9lJbj9qmuPvD7BwNUl6wms' # registry with backers, no signatures. should escrow with basing.openDB() as db, keeping.openKS() as kpr, viring.openReger() as reg: @@ -489,17 +398,16 @@ def test_tever_escrow(mockCoringRandomNonce): dgkey = dgKey(pre=regk, dig=vcp.said) vcp = reg.getTvt(dgkey) - assert bytes(vcp) == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EGMjDTKwtL24UE06lU1ZR6LGBQFZTRa5bFiA' - b'3WDTa8GO","i":"EGMjDTKwtL24UE06lU1ZR6LGBQFZTRa5bFiA3WDTa8GO","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BAOcciw30' - b'IVQsaenKXpiyMVrjtPDW3KeD_6KFnSfoaqI"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') + assert bytes(vcp) == (b'{"v":"KERI10JSON000129_","t":"vcp","d":"EBkUjPBzZuFeSTP-Quuz0Exr6jdUNd8VDa5h' + b'oNvnS1Jo","i":"EBkUjPBzZuFeSTP-Quuz0Exr6jdUNd8VDa5hoNvnS1Jo","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BAOcciw30' + b'IVQsaenKXpiyMVrjtPDW3KeD_6KFnSfoaqI"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') anc = reg.getAnc(dgkey) - assert bytes(anc) == b'0AAAAAAAAAAAAAAAAAAAAAABEAfW0zq1WbQ7e6fYQ8EvEZUpZ6nn1RX_Zjmczeke6JTx' + assert bytes(anc) == b'0AAAAAAAAAAAAAAAAAAAAAABEErlKvdXWTpOg9s5AasfABIk17o_EaM2uAuGe-c_LEbc' assert reg.getTel(snKey(pre=regk, sn=0)) is None dig = reg.getTwe(snKey(pre=regk, sn=0)) - assert bytes(dig) ==b'EGMjDTKwtL24UE06lU1ZR6LGBQFZTRa5bFiA3WDTa8GO' + assert bytes(dig) ==b'EBkUjPBzZuFeSTP-Quuz0Exr6jdUNd8VDa5hoNvnS1Jo' def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): @@ -531,14 +439,15 @@ def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): assert tev.sn == 0 dgkey = dgKey(pre=regk, dig=vcp.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000113_","t":"vcp","d":"ECRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taW' - b'GplUj03s","i":"ECRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":["NB"],"bt":"0","b":[],"n":' - b'"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK3SM-S8a8Y_U"}') + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON0000ff_","t":"vcp","d":"EKWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZ' + b'XdRB2g6C","i":"EKWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":["NB"],"bt":"0","b":[],"n":' + b'"0AAUiJMii_rPXXCiLTEEaDT7"}') - assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABEFEUzOixM-Avz6VPmQoB59eYQ2oan9ltJ1JOSe-QQFRq' - assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'ECRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s' + assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABEGe5uFh3t0JglSPQtJJoxCV1RlFoTBept1BPRk3o6hgh' + assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'EKWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C' assert reg.getTibs(dgkey) == [] assert reg.getTwe(snKey(pre=regk, sn=0)) is None @@ -571,11 +480,12 @@ def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): vci = vcdig dgkey = dgKey(pre=vci, dig=iss.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON0000ed_","t":"iss","d":"EPlZDm4GgNl2aZNvHyfbk-B8mN6dNMMdJxrR' - b'LovbJSCS","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"0","ri":"E' - b'CRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s","dt":"2021-01-01T00:00:00.00000' - b'0+00:00"}') - assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAADEOMHrVBAHEJ-5qHzoBLLvBj_m9761m7CavMZmVOXgZpT' + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON0000ed_","t":"iss","d":"EFQdb41xs1FwGz0m-Ekzwv9gwnpD8hKc4XGJ' + b'3-jPUA6I","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"0","ri":"E' + b'KWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C","dt":"2021-01-01T00:00:00.00000' + b'0+00:00"}') + assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAADEHzX4VXW_xCDf_pFhFvkDdc6TXTxiSiaEpHHCr1tDVtD' # revoke vc with no backers rev = eventing.revoke(vcdig=vcdig.decode("utf-8"), regk=regk, dig=iss.said) @@ -590,10 +500,11 @@ def test_tever_no_backers(mockHelpingNowUTC, mockCoringRandomNonce): tev.update(rev, seqner=seqner, saider=saider) dgkey = dgKey(pre=vci, dig=rev.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000120_","t":"rev","d":"EGjtu2bIII28dwxA0BH8KeXCN03U7TN3SkLD' - b'1KZi77pj","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"1","ri":"E' - b'CRIn7b_Y6aB-G3x45-fPb_LQgke8CDq0taWGplUj03s","p":"EPlZDm4GgNl2aZNvHyfbk-B8mN' - b'6dNMMdJxrRLovbJSCS","dt":"2021-01-01T00:00:00.000000+00:00"}') + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON000120_","t":"rev","d":"EEXxqWHCGw1XAkzEX_32xyRSboJDIwCKZUA9' + b'WfBzn-jx","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","s":"1","ri":"E' + b'KWuqbpBPglFWnzZuD3f_DTCLwYd4ub1bWUZXdRB2g6C","p":"EFQdb41xs1FwGz0m-Ekzwv9gwn' + b'pD8hKc4XGJ3-jPUA6I","dt":"2021-01-01T00:00:00.000000+00:00"}') # assert reg.getAnc(dgkey) == b'0AAAAAAAAAAAAAAAAAAAAABAECgc6yHeTRhsKh1M7k65feWZGCf_MG0dWoei5Q6SwgqU' @@ -631,15 +542,15 @@ def test_tever_backers(mockHelpingNowUTC, mockCoringRandomNonce): tev = Tever(serder=vcp, seqner=seqner, saider=saider, bigers=[valCigar], db=db, reger=reg) dgkey = dgKey(pre=regk, dig=vcp.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON00013d_","t":"vcp","d":"EBgdJt_ASWeq7HjOmut2E8vQL8P1c9VTPDA0' - b'Pdh4KsZX","i":"EBgdJt_ASWeq7HjOmut2E8vQL8P1c9VTPDA0Pdh4KsZX","ii":"EPst_DQ1d' - b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BPmRWtx8n' - b'wSzRdJ0zTvP5uBb0t3BSjjstDk0gTayFfjV"],"n":"A9XfpxIl1LcIkMhUSCCC8fgvkuX8gG9xK' - b'3SM-S8a8Y_U"}') - assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABELUmsgBmQ-OwMp2Zi7NpnRvZPwcWqld49zvTP9r-I4vp' - assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'EBgdJt_ASWeq7HjOmut2E8vQL8P1c9VTPDA0Pdh4KsZX' - assert [bytes(tib) for tib in reg.getTibs(dgkey)] == [b'AAAzew389hwVg7TzgHucIdznjjgWh9A9w3T2YAe6A1U7mQCG_tEhQ_px7G0ZO39GdtyDx4Z890e0' - b'WBPKdTyKbqIE'] + assert bytes(reg.getTvt(dgkey)) == ( + b'{"v":"KERI10JSON000129_","t":"vcp","d":"ECfzJv1hIYAF68tEDDSelka5aPNKg_pmdcZO' + b'Ts0aubF-","i":"ECfzJv1hIYAF68tEDDSelka5aPNKg_pmdcZOTs0aubF-","ii":"EPst_DQ1d' + b'8VCMGHB475dgKWCxO3qX4HlvW_4_lsrVZ9Q","s":"0","c":[],"bt":"1","b":["BPmRWtx8n' + b'wSzRdJ0zTvP5uBb0t3BSjjstDk0gTayFfjV"],"n":"0AAUiJMii_rPXXCiLTEEaDT7"}') + assert bytes(reg.getAnc(dgkey)) == b'0AAAAAAAAAAAAAAAAAAAAAABEDD2vrz4Eg3iLOOlZw5-d3ioZ1q703IC0M0LJP_3v-PT' + assert bytes(reg.getTel(snKey(pre=regk, sn=0))) == b'ECfzJv1hIYAF68tEDDSelka5aPNKg_pmdcZOTs0aubF-' + assert [bytes(tib) for tib in reg.getTibs(dgkey)] == [b'AAAUr5RHYiDH8RU0ig-2Dp5h7rVKx89StH5M3CL60-cWEbgG-XmtW31pZlFicYgSPduJZUnD838_' + b'QLbASSQLAZcC'] assert reg.getTwe(snKey(pre=regk, sn=0)) is None debSecret = 'AKUotEE0eAheKdDJh9QvNmSEmO_bjIav8V_GmctGpuCQ' @@ -685,11 +596,11 @@ def test_tever_backers(mockHelpingNowUTC, mockCoringRandomNonce): vci = vcdig dgkey = dgKey(pre=vci, dig=bis.said) - assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000162_","t":"bis","d":"EN01O_jV46iSPtJMFXLNB83OWHtUl0wEDnMT' - b'eHSWXJwf","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","ii":"EBgdJt_AS' - b'Weq7HjOmut2E8vQL8P1c9VTPDA0Pdh4KsZX","s":"0","ra":{"i":"EBgdJt_ASWeq7HjOmut2' - b'E8vQL8P1c9VTPDA0Pdh4KsZX","s":"1","d":"EEc1UURU3liIomWOFeDVqCo-3QbZHFZCUorx8' - b'MdZFZvy"},"dt":"2021-01-01T00:00:00.000000+00:00"}') + assert bytes(reg.getTvt(dgkey)) == (b'{"v":"KERI10JSON000162_","t":"bis","d":"ECtlJanGHVM3-xcy7nrw06FyqqM-OZaewC28' + b'3XpVjep3","i":"EEBp64Aw2rsjdJpAR0e2qCq3jX7q7gLld3LjAwZgaLXU","ii":"ECfzJv1hI' + b'YAF68tEDDSelka5aPNKg_pmdcZOTs0aubF-","s":"0","ra":{"i":"ECfzJv1hIYAF68tEDDSe' + b'lka5aPNKg_pmdcZOTs0aubF-","s":"1","d":"EOP4g7675ItLSVGbYsmpalH_UBQaJE-Ir5I83' + b'carNegC"},"dt":"2021-01-01T00:00:00.000000+00:00"}') def test_tevery(): @@ -777,7 +688,7 @@ def test_tevery_process_escrow(mockCoringRandomNonce): seqner = Seqner(sn=1) # said of rotation - rotsaid = b'EFEUzOixM-Avz6VPmQoB59eYQ2oan9ltJ1JOSe-QQFRq' + rotsaid = b'EGe5uFh3t0JglSPQtJJoxCV1RlFoTBept1BPRk3o6hgh' diger = coring.Diger(qb64b=rotsaid) tvy = Tevery(reger=reg, db=db) @@ -802,4 +713,5 @@ def test_tevery_process_escrow(mockCoringRandomNonce): if __name__ == "__main__": #test_tever_escrow() #test_tevery_process_escrow() - test_prefixer() + pass + diff --git a/tests/vdr/test_reger_mapsize.py b/tests/vdr/test_reger_mapsize.py new file mode 100644 index 000000000..df9c4bb47 --- /dev/null +++ b/tests/vdr/test_reger_mapsize.py @@ -0,0 +1,54 @@ +# -*- encoding: utf-8 -*- +""" +tests.vdr.test_reger_mapsize module + +""" +import os +import pytest + + +def test_reger_specific_env_var(): + from keri.vdr import viring + + os.environ['KERI_REGER_MAP_SIZE'] = '150000000' + + try: + reger = viring.Reger(name='test_reger', temp=True) + assert reger.MapSize == 150000000 + reger.close() + finally: + os.environ.pop('KERI_REGER_MAP_SIZE', None) + + +def test_reger_general_env_var_fallback(): + from keri.vdr import viring + + os.environ['KERI_LMDB_MAP_SIZE'] = '250000000' + + try: + reger = viring.Reger(name='test_reger_general', temp=True) + assert reger.MapSize == 250000000 + reger.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + + +def test_reger_specific_takes_precedence(): + from keri.vdr import viring + + os.environ['KERI_LMDB_MAP_SIZE'] = '100000000' + os.environ['KERI_REGER_MAP_SIZE'] = '200000000' + + try: + reger = viring.Reger(name='test_reger_precedence', temp=True) + assert reger.MapSize == 200000000 + reger.close() + finally: + os.environ.pop('KERI_LMDB_MAP_SIZE', None) + os.environ.pop('KERI_REGER_MAP_SIZE', None) + + +if __name__ == '__main__': + test_reger_specific_env_var() + test_reger_general_env_var_fallback() + test_reger_specific_takes_precedence() diff --git a/tests/vdr/test_txn_state.py b/tests/vdr/test_txn_state.py index fa24dbe2f..8daa46b0e 100644 --- a/tests/vdr/test_txn_state.py +++ b/tests/vdr/test_txn_state.py @@ -1,9 +1,12 @@ from dataclasses import asdict -from keri.app import habbing +from keri import core from keri.core import routing, parsing, coring, serdering + from keri.core.eventing import Kevery, SealEvent +from keri.app import habbing + from keri.vc import proving from keri.vdr import viring, credentialing, eventing @@ -12,11 +15,13 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -29,7 +34,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # Gather up Bob's key event log msgs = bytearray() @@ -48,11 +53,11 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(rsr) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', 's': '0', 'vn': [1, 0]} @@ -69,7 +74,7 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): assert cue['q']['ri'] == issuer.regk saider = bamReger.txnsb.escrowdb.get(keys=("registry-ooo", issuer.regk, bobHab.pre)) - assert saider[0].qb64b == b'ELprJNCkha3b3t7YKOPzfv-prTZvgJFM_xTnaaJop-3A' + assert saider[0].qb64b == b'ECZWYxq_Qgs0J0ls_imRWRYxrojzTKL2REjqe0rN8kWy' tmsgs = bytearray() cloner = regery.reger.clonePreIter(pre=issuer.regk, fn=0) # create iterator at 0 @@ -84,17 +89,18 @@ def test_tsn_message_out_of_order(mockHelpingNowUTC, mockCoringRandomNonce): assert bamReger.txnsb.escrowdb.get(keys=(issuer.regk, bobHab.pre)) == [] # check to make sure the tsn has been saved saider = bamReger.txnsb.saiderdb.get(keys=(issuer.regk, bobHab.pre)) - assert saider.qb64b == b'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert saider.qb64b == b'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -107,7 +113,7 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # pass key event log to Bam bamRtr = routing.Router() @@ -120,11 +126,11 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(tsn) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', 's': '0', 'vn': [1, 0]} @@ -136,7 +142,7 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy) saider = bamReger.txnsb.escrowdb.get(keys=("registry-mae", issuer.regk, bobHab.pre)) - said = b'ELprJNCkha3b3t7YKOPzfv-prTZvgJFM_xTnaaJop-3A' + said = b'ECZWYxq_Qgs0J0ls_imRWRYxrojzTKL2REjqe0rN8kWy' assert saider[0].qb64b == said assert len(bamTvy.cues) == 1 cue = bamTvy.cues.popleft() @@ -174,28 +180,25 @@ def test_tsn_message_missing_anchor(mockHelpingNowUTC, mockCoringRandomNonce): assert bamReger.txnsb.escrowdb.get(keys=(issuer.regk, bobHab.pre)) == [] # check to make sure the tsn has been saved saider = bamReger.txnsb.saiderdb.get(keys=(issuer.regk, bobHab.pre)) - assert saider.qb64b == b'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert saider.qb64b == b'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): # Bob is the controller # Wes is his witness # Bam is verifying the key state for Bob from Wes - #raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - #salter = coring.Salter(raw=raw) - #salt = salter.qb64 - #assert salt == '0AAFqo8tU5rp-lWcApybCEh1' # Habery.makeHab uses name as stem path for salt so different pre - with (habbing.openHby(name="wes", base="test") as wesHby, - habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, + habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1,transferable=False,) - assert wesHab.pre == 'BCuDiSPCTq-qBBFDHkhf1_kmysrH8KSsFvoaOSgEbx-X' + assert wesHab.pre == 'BJX05FKbj6M7EoUp53nKJNdG5eDZMGBatlDjg_QcpuqE' bobHab = bobHby.makeHab(name="bob", isith='1', icount=1, wits=[wesHab.pre]) - assert bobHab.pre == 'EDroh9lTel0P1YQaiL7shXG63SRSzKSDek7PaceOs6bY' + assert bobHab.pre == 'EPa6GLVG4lFV9oi28WQbC7UfcSoDb7kMlZkaa3qaj4UA' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -208,13 +211,13 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP' + assert issuer.regk == 'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6' # Create Bob's icp, pass to Wes. wesKvy = Kevery(db=wesHby.db, lax=False, local=False) for msg in bobHby.db.clonePreIter(pre=bobHab.pre, fn=0): - parsing.Parser().parse(ims=bytearray(msg), kvy=wesKvy) + parsing.Parser().parse(ims=bytearray(msg), kvy=wesKvy, local=True) iserder = serdering.SerderKERI(raw=bytearray(msg)) wesHab.receipt(serder=iserder) @@ -240,11 +243,11 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(tsn) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP', + 'd': 'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP', - 'ii': 'EDroh9lTel0P1YQaiL7shXG63SRSzKSDek7PaceOs6bY', + 'i': 'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6', + 'ii': 'EPa6GLVG4lFV9oi28WQbC7UfcSoDb7kMlZkaa3qaj4UA', 's': '0', 'vn': [1, 0]} @@ -256,10 +259,10 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): bamReger = viring.Reger(name="bam", temp=True) bamTvy = eventing.Tevery(reger=bamReger, db=bamHby.db, lax=False, local=False, rvy=bamRvy) bamTvy.registerReplyRoutes(router=bamRtr) - parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy, local=True) saider = bamReger.txnsb.escrowdb.get(keys=("registry-mae", issuer.regk, wesHab.pre)) - said = b'EMoqBJpoPJCkCejSUZ8IShTeGd_WQkF0zOQc2l0HQusn' + said = b'EPKAvk7JJzylDFPjOxfanPjqDvUk_a-hzjNTT9Svhik1' assert saider[0].qb64b == said assert len(bamTvy.cues) == 1 cue = bamTvy.cues.popleft() @@ -267,14 +270,14 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): assert cue['q']['pre'] == bobHab.pre wesIcp = wesHab.makeOwnEvent(sn=0) - parsing.Parser().parse(ims=bytearray(wesIcp), kvy=bamKvy) + parsing.Parser().parse(ims=bytearray(wesIcp), kvy=bamKvy, local=True) assert wesHab.pre in bamHby.db.kevers msgs = bytearray() for msg in wesHby.db.clonePreIter(pre=bobHab.pre, fn=0): msgs.extend(msg) - parsing.Parser().parse(ims=msgs, kvy=bamKvy, rvy=bamRvy) + parsing.Parser().parse(ims=msgs, kvy=bamKvy, rvy=bamRvy, local=True) assert bobHab.pre in bamHby.db.kevers bamTvy.processEscrows() @@ -287,7 +290,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): saider = bamReger.txnsb.escrowdb.get(keys=("registry-ooo", issuer.regk, wesHab.pre)) assert saider[0].qb64b == said - parsing.Parser().parse(ims=bytearray(tmsgs), tvy=bamTvy, rvy=bamRvy) + parsing.Parser().parse(ims=bytearray(tmsgs), tvy=bamTvy, rvy=bamRvy, local=True) assert issuer.regk in bamReger.tevers @@ -297,7 +300,7 @@ def test_tsn_from_witness(mockHelpingNowUTC, mockCoringRandomNonce): assert bamReger.txnsb.escrowdb.get(keys=(issuer.regk, wesHab.pre)) == [] # check to make sure the tsn has been saved saider = bamReger.txnsb.saiderdb.get(keys=(issuer.regk, wesHab.pre)) - assert saider.qb64b == b'EBrr1pxZoY5nY38YifrGvn5HSMv0sAwvTTAQ5e3_-ivP' + assert saider.qb64b == b'EH3hN33719ybSg21Kboy-V2jafwvQSHnY1HUGzzBqqk6' def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): @@ -305,20 +308,21 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): # Bam is verifying the key state for Bob from Wes # Wes is no one #raw = b'\x05\xaa\x8f-S\x9a\xe9\xfaU\x9c\x02\x9c\x9b\x08Hu' - #salter = coring.Salter(raw=raw) + #salter = core.Salter(raw=raw) #salt = salter.qb64 #assert salt == '0AAFqo8tU5rp-lWcApybCEh1' # Habery.makeHab uses name as stem path for salt so different pre - with (habbing.openHby(name="wes", base="test") as wesHby, - habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="wes", base="test", salt=default_salt) as wesHby, + habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): # setup Wes's habitat nontrans wesHab = wesHby.makeHab(name="wes", isith='1', icount=1,transferable=False,) - assert wesHab.pre == 'BCuDiSPCTq-qBBFDHkhf1_kmysrH8KSsFvoaOSgEbx-X' + assert wesHab.pre == 'BJX05FKbj6M7EoUp53nKJNdG5eDZMGBatlDjg_QcpuqE' bobHab = bobHby.makeHab(name="bob", isith='1', icount=1) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -331,7 +335,7 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # Create Bob's icp, pass to Wes. @@ -362,11 +366,11 @@ def test_tsn_from_no_one(mockHelpingNowUTC, mockCoringRandomNonce): assert asdict(tsn) == {'b': [], 'bt': '0', 'c': ['NB'], - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 'dt': '2021-01-01T00:00:00.000000+00:00', 'et': 'vcp', - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', 's': '0', 'vn': [1, 0]} @@ -401,11 +405,12 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe # Bob is the controller # Bam is verifying the key state for Bob with a stale key state in the way - with (habbing.openHby(name="bob", base="test") as bobHby, - habbing.openHby(name="bam", base="test") as bamHby): + default_salt = core.Salter(raw=b'0123456789abcdef').qb64 + with (habbing.openHby(name="bob", base="test", salt=default_salt) as bobHby, + habbing.openHby(name="bam", base="test", salt=default_salt) as bamHby): bobHab = bobHby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=bobHby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -418,7 +423,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' # pass key event log to Bam bamRtr = routing.Router() @@ -448,25 +453,25 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe tever = issuer.tevers[issuer.regk] rsr = tever.state() - assert rsr._asdict() == {'vn': [1, 0], - 'i': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 's': '0', - 'd': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', - 'ii': 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH', - 'dt': '2021-06-27T21:26:21.233257+00:00', - 'et': 'vcp', - 'bt': '0', - 'b': [], - 'c': ['NB']} + assert rsr._asdict() == {'b': [], + 'bt': '0', + 'c': ['NB'], + 'd': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'dt': '2021-06-27T21:26:21.233257+00:00', + 'et': 'vcp', + 'i': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', + 'ii': 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj', + 's': '0', + 'vn': [1, 0]} ctsn = tever.vcState(vci=creder.said) - assert asdict(ctsn) == {'a': {'d': 'EGJIFzWexy6LQbW4-IQqhGLD6wA9yR7pcLzxomjb40Ku', 's': 2}, - 'd': 'EIxhyBA8h6BMmtWEJzqNkoAquIkMucpXbdY3kQX25GQu', + assert asdict(ctsn) == {'a': {'d': 'EJEsBdnprtcNR-QaFM-h1OPaiGDBikgDk7aqVXPHAhPL', 's': 2}, + 'd': 'EGE5B5vYhHNYHN0g8YHIcAIu3LR0vJQ9fCXqjJunEZf1', 'dt': '2021-06-27T21:26:21.233257+00:00', 'et': 'iss', - 'i': 'EEqcwL-ew_OaQphSQvy8bRGtnKlL_g_SkXjsAGWgtFGl', + 'i': 'EPQNM4YNlVZ6nGu7ElGC6GhAHdLBLnoxYXKFP4R_ZbGz', 'ra': {}, - 'ri': 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6', + 'ri': 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ', 's': '0', 'vn': [1, 0]} @@ -478,7 +483,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe parsing.Parser().parse(ims=bytearray(rpy), tvy=bamTvy, rvy=bamRvy) saider = bamReger.txnsb.escrowdb.get(keys=("credential-mre", creder.said, bobHab.pre)) - assert saider[0].qb64b == b'EAslzIz1FKVBY1zd_0gF-gclpvHMf1V4arW3puZlPv4K' + assert saider[0].qb64b == b'EHERhBLfaMik0Ne9ysU3UICXWge0yobK0FQv3QhyeqF7' assert len(bamTvy.cues) == 1 cue = bamTvy.cues.popleft() assert cue["kin"] == "telquery" @@ -507,7 +512,7 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe assert cue['q']['ri'] == issuer.regk saider = bamReger.txnsb.escrowdb.get(keys=("credential-ooo", creder.said, bobHab.pre)) - assert saider[0].qb64b == b'EAslzIz1FKVBY1zd_0gF-gclpvHMf1V4arW3puZlPv4K' + assert saider[0].qb64b == b'EHERhBLfaMik0Ne9ysU3UICXWge0yobK0FQv3QhyeqF7' vci = creder.said tmsgs = bytearray() @@ -524,15 +529,13 @@ def test_credential_tsn_message(mockHelpingNowUTC, mockCoringRandomNonce, mockHe # check to make sure the tsn has been saved keys = (creder.said, bobHab.pre) saider = bamReger.txnsb.saiderdb.get(keys=keys) - assert saider.qb64b == b'EIxhyBA8h6BMmtWEJzqNkoAquIkMucpXbdY3kQX25GQu' + assert saider.qb64b == b'EGE5B5vYhHNYHN0g8YHIcAIu3LR0vJQ9fCXqjJunEZf1' def test_tever_reload(mockHelpingNowUTC, mockCoringRandomNonce, mockHelpingNowIso8601): - - with habbing.openHby(name="bob", base="test") as hby: - + with habbing.openHby(name="bob", base="test", salt=core.Salter(raw=b'0123456789abcdef').qb64) as hby: bobHab = hby.makeHab(name="bob", isith='1', icount=1,) - assert bobHab.pre == 'EA_SbBUZYwqLVlAAn14d6QUBQCSReJlZ755JqTgmRhXH' + assert bobHab.pre == 'EFggrgspyZwbi-zB2iJzjHu0QU5dh89mA8jOhDcgrTqj' regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=bobHab.pre, name=bobHab.name) @@ -545,7 +548,7 @@ def test_tever_reload(mockHelpingNowUTC, mockCoringRandomNonce, mockHelpingNowIs saider=coring.Saider(qb64=bobHab.kever.serder.said)) regery.processEscrows() - assert issuer.regk == 'ECbNKwkTjZqsfwNLxTnraPImegy1YeQ2-pCrTBQmu3i6' + assert issuer.regk == 'EClqKVJREM3MWKBqR2j712s3Z6rPxhqO-h-p8Ls6_9hQ' rsr = regery.reger.states.get(keys=issuer.regk) tever = eventing.Tever(rsr=rsr, reger=regery.reger) diff --git a/tests/vdr/test_verifying.py b/tests/vdr/test_verifying.py index a0e8bbf3c..6cd16154e 100644 --- a/tests/vdr/test_verifying.py +++ b/tests/vdr/test_verifying.py @@ -9,7 +9,7 @@ from keri import kering from keri.app import habbing, signing from keri.core import eventing as ceventing, scheming -from keri.core import parsing, coring +from keri.core import parsing, coring, indexing from keri.core.eventing import SealEvent from keri.help import helping from keri.vc import proving @@ -17,7 +17,7 @@ def test_verifier_query(mockHelpingNowUTC, mockCoringRandomNonce): - with habbing.openHab(name="test", transferable=True, temp=True) as (hby, hab): + with habbing.openHab(name="test", transferable=True, temp=True, salt=b'0123456789abcdef') as (hby, hab): regery = credentialing.Regery(hby=hby, name="test", temp=True) issuer = regery.makeRegistry(prefix=hab.pre, name="test") @@ -25,13 +25,13 @@ def test_verifier_query(mockHelpingNowUTC, mockCoringRandomNonce): msg = verfer.query(hab.pre, issuer.regk, "EA8Ih8hxLi3mmkyItXK1u55cnHl4WgNZ_RE-gKXqgcX4", route="tels") - assert msg == (b'{"v":"KERI10JSON0000fe_","t":"qry","d":"EHraBkp-XMf1x_bo70O2x3br' - b'BCHlJHa7q_MzsBNeYz2_","dt":"2021-01-01T00:00:00.000000+00:00","r' + assert msg == (b'{"v":"KERI10JSON0000fe_","t":"qry","d":"EFa6oMZA5bgpALIc7yykT6O6' + b'ovdbDQnRFeTPDI4zaOhr","dt":"2021-01-01T00:00:00.000000+00:00","r' b'":"tels","rr":"","q":{"i":"EA8Ih8hxLi3mmkyItXK1u55cnHl4WgNZ_RE-g' - b'KXqgcX4","ri":"EO0_SyqPS1-EVYSITakYpUHaUZZpZGsjaXFOaO_kCfS4"}}-V' - b'Aj-HABEIaGMMWJFPmtXznY1IIiKDIrg-vIyge6mBl2QV8dDjI3-AABAABUWETZTw' - b'TVuh0mNvN5KsJ_9V1epoP5wqgW32x8nUnGB20aI8xQBAhQ-aVP61ZEq97BDGSnxO' - b'hU6tGCfDmvtugI') + b'KXqgcX4","ri":"EB-u4VAF7A7_GR8PXJoAVHv5X9vjtXew8Yo6Z3w9mQUQ"}}-V' + b'Aj-HABEMl4RhuR_JxpiMd1N8DEJEhTxM3Ovvn9Xya8AN-tiUbl-AABAABGnrnayV' + b'yK1siivaffGHpWWhcVThPN_dsePQvMXrlsOYNf0UdT0e6ch-0bN-UuOJCd1behue' + b'Zs_0V9FQ9vw0wK') def test_verifier(seeder): @@ -115,6 +115,9 @@ def test_verifier(seeder): for idx, cred in enumerate(creds): assert dcre.sad == cred["sad"] + with pytest.raises(kering.MissingEntryError): + regery.reger.cloneCred(said="nonexistantsaid") + """End Test""" @@ -175,7 +178,7 @@ def test_verifier(seeder): # sigs.extend(hab2.db.getSigs(dgkey)) # sigs.extend(hab3.db.getSigs(dgkey)) # -# sigers = [coring.Siger(qb64b=bytes(sig)) for sig in sigs] +# sigers = [indexing.Siger(qb64b=bytes(sig)) for sig in sigs] # # evt = bytearray(eraw) # evt.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, @@ -252,7 +255,7 @@ def test_verifier(seeder): # sigs.extend(hab2.db.getSigs(dgkey)) # sigs.extend(hab3.db.getSigs(dgkey)) # -# sigers = [coring.Siger(qb64b=bytes(sig)) for sig in sigs] +# sigers = [indexing.Siger(qb64b=bytes(sig)) for sig in sigs] # # evt = bytearray(eraw) # evt.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, @@ -310,8 +313,8 @@ def test_verifier_chained_credential(seeder): with habbing.openHab(name="ron", temp=True, salt=b'0123456789abcdef') as (ronHby, ron), \ habbing.openHab(name="ian", temp=True, salt=b'0123456789abcdef') as (ianHby, ian), \ - habbing.openHab(name="han", transferable=True, temp=True) as (hanHby, han), \ - habbing.openHab(name="vic", transferable=True, temp=True) as (vicHby, vic): + habbing.openHab(name="han", transferable=True, temp=True, salt=b'0123456789abcdef') as (hanHby, han), \ + habbing.openHab(name="vic", transferable=True, temp=True, salt=b'0123456789abcdef') as (vicHby, vic): seeder.seedSchema(db=ronHby.db) seeder.seedSchema(db=ianHby.db) seeder.seedSchema(db=hanHby.db) diff --git a/tests/vdr/test_viring.py b/tests/vdr/test_viring.py index dd84c1448..c7e461634 100644 --- a/tests/vdr/test_viring.py +++ b/tests/vdr/test_viring.py @@ -8,9 +8,14 @@ import os import lmdb - -from keri.core.coring import Diger, versify, Serials -from keri.db.dbing import openLMDB, dgKey, snKey +import pytest + +from keri import kering +from keri.app.keeping import KERIKeeperMapSizeKey +from keri.core import coring +from keri.core.coring import Diger, versify, Kinds +from keri.core.serdering import SerderACDC +from keri.db.dbing import openLMDB, dgKey, snKey, KERIBaserMapSizeKey, KERIRegerMapSizeKey, LMDBer from keri.vdr.viring import Reger @@ -77,7 +82,7 @@ def test_issuer(): # test with registry inception (vcp) event regk = regb sn = 0 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) vcp = dict(v=vs, i=regk.decode("utf-8"), s="{:x}".format(sn), b=[rarb.decode("utf-8")], @@ -184,7 +189,7 @@ def test_issuer(): # test with verifiable credential issuance (iss) event vcdig = b'EAvR3p8V95W8J7Ui4-mEzZ79S-A1esAnJo1Kmzq80Jkc' sn = 0 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) vcp = dict(v=vs, i=vcdig.decode("utf-8"), s="{:x}".format(sn), @@ -231,7 +236,7 @@ def test_issuer(): assert issuer.putTel(snKey(vcdig, sn + 2), val=idig.qb64b) is True assert issuer.putTel(snKey(vcdig, sn + 3), val=rdig.qb64b) is True - result = [(sn, dig) for sn, dig in issuer.getTelItemPreIter(vcdig)] + result = [(sn, dig) for _, sn, dig in issuer.getTelItemPreIter(vcdig)] assert result == [(0, idig.qb64b), (1, rdig.qb64b), (2, idig.qb64b), (3, rdig.qb64b)] bak1 = b'BA1Q98kT0HRn9R62lY-LufjjKdbCeL1mqu9arTgOmbqI' @@ -267,7 +272,7 @@ def test_clone(): # test with registry inception (vcp) event sn = 0 - vs = versify(kind=Serials.json, size=20) + vs = versify(kind=Kinds.json, size=20) vcp = dict(v=vs, i=regk.decode("utf-8"), s="{:x}".format(sn), b=[rarb.decode("utf-8")], @@ -341,6 +346,114 @@ def test_clone(): b'PjioY7Ycna6ouhSSH0QcKsEjce10HCXIW_XtmEYr9SrB5BA-GAB0AAAAAAAAAAAA' b'AAAAAAAAABCEzpq06UecHwzy-K9FpNoRxCJp2wIGM9u2Edk-PLMZ1H4') +def test_clearEscrows(): + with openLMDB(cls=Reger) as db: + regk = "EAWdT7a7fZwRz0jiZ0DJxZEM3vsNbLDPEUk-ODnif3O0".encode("utf-8") + sn = 0 + ooKey = snKey(regk, sn) + vs = versify(kind=coring.Kinds.json, size=20) + rarb = "BBjzaUuRMwh1ivT5BQrqNhbvx82lB-ofrHVHjL3WADbA".encode("utf-8") + vcp = dict(v=vs, i=regk.decode("utf-8"), + s="{:x}".format(sn), b=[rarb.decode("utf-8")], + t="vcp") + + vcpb = json.dumps(vcp, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + vdig = Diger(ser=vcpb) + + db.putOot(ooKey, val=vdig.qb64b) + db.putTwe(ooKey, val=vdig.qb64b) + db.putTae(ooKey, val=vdig.qb64b) + + pre = 'k' + db.mre.put(keys=(pre, ), val=coring.Dater()) + db.mce.put(keys=(pre, ), val=coring.Dater()) + db.mse.put(keys=(pre, ), val=coring.Dater()) + + creder = SerderACDC(makify=True, proto=kering.Protocols.acdc, verify=False) + db.cmse.put(("a", "b"), val=creder) + + prefixer = coring.Prefixer(qb64="EAD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I") + seqner = coring.Seqner(sn=0) + saider = coring.Saider(qb64="EAD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I") + db.tpwe.add(("a", "b"), (prefixer, seqner, saider)) + db.tmse.add(("a", "b"), (prefixer, seqner, saider)) + db.tede.add(("a", "b"), (prefixer, seqner, saider)) + + dater = coring.Dater() + broker_saider = coring.Saider(qb64="EBD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I") + aid = "ECD919wF4oiG7ck6mnBWTRD_Z-Io0wZKCxL0zjx5je9I" + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + db.txnsb.escrowdb.put(keys=(typ, pre, aid), vals=[broker_saider]) + db.txnsb.daterdb.put(keys=(broker_saider.qb64,), val=dater) + + assert db.getOot(ooKey) == vdig.qb64b + assert db.getTwe(ooKey) == vdig.qb64b + assert db.getTae(ooKey) == vdig.qb64b + assert db.mre.get((pre, )) is not None + assert db.mce.get((pre, )) is not None + assert db.mse.get((pre, )) is not None + assert db.cmse.cntAll() == 1 + assert db.tpwe.cntAll() == 1 + assert db.tmse.cntAll() == 1 + assert db.tede.cntAll() == 1 + + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + assert db.txnsb.escrowdb.get(keys=(typ, pre, aid)) != [] + + db.clearEscrows() + + assert db.getOot(ooKey) is None + assert db.getTwe(ooKey) is None + assert db.getTae(ooKey) is None + assert db.mre.get((pre, )) is None + assert db.mce.get((pre, )) is None + assert db.mse.get((pre, )) is None + assert db.cmse.cntAll() == 0 + assert db.tpwe.cntAll() == 0 + assert db.tmse.cntAll() == 0 + assert db.tede.cntAll() == 0 + + # Verify Broker escrows were cleared + for typ in ["registry-mae", "registry-ooo", "credential-mre", "credential-mae", "credential-ooo"]: + assert db.txnsb.escrowdb.get(keys=(typ, pre, aid)) == [] + assert db.txnsb.daterdb.get(keys=(broker_saider.qb64,)) is None + +def test_mailbox_db_size_set_from_env_var(): + # Clear environment before test + if KERIBaserMapSizeKey in os.environ: + os.environ.pop(KERIBaserMapSizeKey) + if KERIRegerMapSizeKey in os.environ: + os.environ.pop(KERIRegerMapSizeKey) + + new_map_size = 10737418240 + # Default map size works + reger = Reger() + assert reger.env.info()['map_size'] != new_map_size, "Expected map size to be the default 10MB" + assert reger.env.info()['map_size'] == LMDBer.MapSize, "Expected map size to be the default 10MB" + reger.close() + + # Specific map size works + os.environ[KERIRegerMapSizeKey] = f"{new_map_size}" + + reger = Reger() + assert reger.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + os.environ.pop(KERIRegerMapSizeKey) + reger.close() + + # generic map size works + baser_map_size = 10737418240 + os.environ[KERIRegerMapSizeKey] = f"{baser_map_size}" + + reger = Reger() + assert reger.env.info()['map_size'] == new_map_size, "Expected map size to be set from environment variable to 10GB" + reger.close() + + # Bad map size throws + os.environ[KERIRegerMapSizeKey] = f"bad_map_size" + with pytest.raises(ValueError) as excinfo: + Reger() + assert "invalid literal for int" in str(excinfo.value), "Expected ValueError when map size is not an integer" + os.environ.pop(KERIRegerMapSizeKey) if __name__ == "__main__": test_issuer()