diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 04e8b283..e2ffead2 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -28,6 +28,11 @@ on: type: string default: all description: "SDK to focus on (go, js, java, all)" + dpop-challenge: + required: false + type: boolean + default: false + description: "Enable DPoP nonce challenge on KAS instances" workflow_call: inputs: platform-ref: @@ -50,6 +55,10 @@ on: required: false type: string default: all + dpop-challenge: + required: false + type: boolean + default: false schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -297,13 +306,14 @@ jobs: ######## SPIN UP PLATFORM BACKEND ############# - name: Check out and start up platform with deps/containers id: run-platform - uses: opentdf/platform/test/start-up-with-containers@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-up-with-containers@0612ea897264e3c621ce076029a5e1e3f2d3971e # main with: platform-ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)[matrix.platform-tag] }} ec-tdf-enabled: true extra-keys: ${{ steps.load-extra-keys.outputs.EXTRA_KEYS }} log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} + dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 @@ -591,7 +601,7 @@ jobs: - name: Start additional kas id: kas-alpha if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@0612ea897264e3c621ce076029a5e1e3f2d3971e # main: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: alpha @@ -599,11 +609,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} - name: Start additional kas id: kas-beta if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@0612ea897264e3c621ce076029a5e1e3f2d3971e # main: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: beta @@ -611,11 +622,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} - name: Start additional kas id: kas-gamma if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@0612ea897264e3c621ce076029a5e1e3f2d3971e # main: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: gamma @@ -623,11 +635,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} - name: Start additional kas id: kas-delta if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@0612ea897264e3c621ce076029a5e1e3f2d3971e # main: require_nonce on additional KAS with: ec-tdf-enabled: true kas-port: 8484 @@ -635,11 +648,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} - name: Start additional KM kas (km1) id: kas-km1 if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@0612ea897264e3c621ce076029a5e1e3f2d3971e # main: require_nonce on additional KAS with: ec-tdf-enabled: true key-management: ${{ steps.km-check.outputs.supported }} @@ -648,11 +662,12 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} - name: Start additional KM kas (km2) id: kas-km2 if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@11af44a5d4826ed281bf2e0e4e31d6ff6154b393 # pqc-enabled + uses: opentdf/platform/test/start-additional-kas@0612ea897264e3c621ce076029a5e1e3f2d3971e # main: require_nonce on additional KAS with: ec-tdf-enabled: true kas-name: km2 @@ -661,6 +676,7 @@ jobs: log-type: json pqc-enabled: ${{ steps.pqc-check.outputs.supported == 'true' }} root-key: ${{ steps.km-check.outputs.root_key }} + dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} - name: Run attribute based configuration tests if: ${{ steps.multikas.outputs.supported == 'true' }} diff --git a/xtest/fixtures/obligations.py b/xtest/fixtures/obligations.py index 953aee05..29a69cbd 100644 --- a/xtest/fixtures/obligations.py +++ b/xtest/fixtures/obligations.py @@ -17,7 +17,8 @@ def otdf_client_scs(otdfctl: OpentdfCommandLineTool) -> abac.SubjectConditionSet: """ Creates a standard subject condition set for OpenTDF clients. - This condition set matches client IDs 'opentdf' or 'opentdf-sdk'. + This condition set matches client IDs 'opentdf', 'opentdf-sdk', or + 'opentdf-dpop' (the DPoP-bound client used by the DPoP tests). Returns: abac.SubjectConditionSet: The created subject condition set @@ -32,7 +33,11 @@ def otdf_client_scs(otdfctl: OpentdfCommandLineTool) -> abac.SubjectConditionSet abac.Condition( subject_external_selector_value=".clientId", operator=abac.SubjectMappingOperatorEnum.IN, - subject_external_values=["opentdf", "opentdf-sdk"], + subject_external_values=[ + "opentdf", + "opentdf-sdk", + "opentdf-dpop", + ], ) ], ) diff --git a/xtest/sdk/java/cli.sh b/xtest/sdk/java/cli.sh index cece9630..aeca8ad3 100755 --- a/xtest/sdk/java/cli.sh +++ b/xtest/sdk/java/cli.sh @@ -42,6 +42,30 @@ else exit 1 fi +# Cache `java -jar cmdline.jar help [...]` output to avoid paying JVM startup +# (typically 150-500ms) for the capability probes on every encrypt/decrypt. +# Keyed by the jar's mtime so a reinstall invalidates the cache. stderr is +# discarded to keep JVM warnings (reflective-access, agent notices) out of logs. +jar_help() { + local jar="$SCRIPT_DIR/cmdline.jar" + local mtime + mtime=$(stat -c %Y "$jar" 2>/dev/null || stat -f %m "$jar" 2>/dev/null || echo 0) + local key + key=$(printf '%s' "$*" | tr -c 'a-zA-Z0-9' '_') + local uid + uid=$(id -u 2>/dev/null || echo default) + local cache="${TMPDIR:-/tmp}/xtest-java-help-${uid}-${mtime}-${key}" + if [[ ! -f "$cache" ]]; then + # Write to a process-unique temp file, then rename: concurrent xdist + # workers see either no cache or the complete file, never a partial read. + local tmp="${cache}.$$" + java -jar "$jar" help "$@" >"$tmp" 2>/dev/null + mv -f "$tmp" "$cache" + fi + cat "$cache" + return 0 +} + if [ "$1" == "supports" ]; then case "$2" in autoconfigure | ns_grants) @@ -118,8 +142,8 @@ if [ "$1" == "supports" ]; then exit $? ;; dpop_nonce_challenge) - echo "dpop_nonce_challenge not supported" - exit 1 + java -jar "$SCRIPT_DIR"/cmdline.jar supports dpop_nonce_challenge + exit $? ;; *) echo "Unknown feature: $2" @@ -135,7 +159,7 @@ args=( ) # when we added support for KAS allowlist, we changed the platform endpoint format to require scheme -if java -jar "$SCRIPT_DIR"/cmdline.jar help decrypt | grep kas-allowlist; then +if jar_help decrypt | grep -q kas-allowlist; then args+=("--platform-endpoint=$PLATFORMURL") else args+=("--platform-endpoint=$PLATFORMENDPOINT") @@ -197,5 +221,9 @@ if [ -n "$XT_WITH_TARGET_MODE" ]; then args+=(--with-target-mode "$XT_WITH_TARGET_MODE") fi +if jar_help | grep -q -- '--verbose'; then + args+=(--verbose) +fi + echo java -jar "$SCRIPT_DIR"/cmdline.jar "${args[@]}" --file="$2" ">" "$3" java -jar "$SCRIPT_DIR"/cmdline.jar "${args[@]}" --file="$2" >"$3" diff --git a/xtest/sdk/js/cli.sh b/xtest/sdk/js/cli.sh index d3160ca3..7e6ded27 100755 --- a/xtest/sdk/js/cli.sh +++ b/xtest/sdk/js/cli.sh @@ -18,6 +18,10 @@ # XT_WITH_ATTRIBUTES [string] - Attributes to be used for encryption # XT_WITH_MIME_TYPE [string] - MIME type for the encrypted file # XT_WITH_TARGET_MODE [string] - Target spec mode for the encrypted file +# XT_WITH_DPOP [string] - Enable DPoP token binding; value selects algorithm (e.g. ES256) +# XT_WITH_DPOP_KEY [string] - Path to PEM-encoded PKCS8 private key for DPoP signing +# CLIENTID [string] - Override OIDC client ID (default: opentdf) +# CLIENTSECRET [string] - Override OIDC client secret (default: secret) # SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) @@ -26,7 +30,7 @@ if grep opentdf/cli "$SCRIPT_DIR/package.json"; then CTL=@opentdf/cli fi -if [ "$1" == "supports" ]; then +if [[ "$1" == "supports" ]]; then if ! cd "$SCRIPT_DIR"; then echo "failed: [cd $SCRIPT_DIR]" exit 1 @@ -112,21 +116,29 @@ if [ "$1" == "supports" ]; then fi XTEST_DIR=$SCRIPT_DIR -while [ "$XTEST_DIR" != "/" ]; do - if [ -f "$XTEST_DIR/pyproject.toml" ] && grep -q 'name = "xtest"' "$XTEST_DIR/pyproject.toml"; then +while [[ "$XTEST_DIR" != "/" ]]; do + if [[ -f "$XTEST_DIR/pyproject.toml" ]] && grep -q 'name = "xtest"' "$XTEST_DIR/pyproject.toml"; then break fi XTEST_DIR=$(dirname "$XTEST_DIR") done -if [ "$XTEST_DIR" = "/" ]; then +if [[ "$XTEST_DIR" = "/" ]]; then echo "xtest root (pyproject.toml with name = \"xtest\") not found." exit 1 fi +# Capture any caller-set overrides before test.env unconditionally resets them. +_pre_clientid="${CLIENTID:-}" +_pre_clientsecret="${CLIENTSECRET:-}" + # shellcheck disable=SC1091 source "$XTEST_DIR"/test.env +# Restore caller overrides (e.g. from pytest monkeypatch for DPoP client). +[[ -n "$_pre_clientid" ]] && CLIENTID="$_pre_clientid" +[[ -n "$_pre_clientsecret" ]] && CLIENTSECRET="$_pre_clientsecret" + src_file=$(realpath "$2") dst_file=$(realpath "$(dirname "$3")")/$(basename "$3") @@ -134,14 +146,21 @@ args=( --output "$dst_file" --kasEndpoint "$KASURL" --oidcEndpoint "$KCFULLURL" - --auth opentdf:secret + --auth "${CLIENTID:-opentdf}:${CLIENTSECRET:-secret}" ) +if [[ -n "$XT_WITH_DPOP" ]]; then + args+=(--dpop "$XT_WITH_DPOP") +fi +if [[ -n "$XT_WITH_DPOP_KEY" ]]; then + args+=(--dpop-key "$XT_WITH_DPOP_KEY") +fi + args+=(--containerType tdf3) -if [ -n "$XT_WITH_ATTRIBUTES" ]; then +if [[ -n "$XT_WITH_ATTRIBUTES" ]]; then attributes="$XT_WITH_ATTRIBUTES" - if [ -f "$attributes" ]; then + if [[ -f "$attributes" ]]; then attributes=$(realpath "$attributes") echo "Attributes are a file: $attributes" args+=(--attributes "$attributes") @@ -152,13 +171,13 @@ if [ -n "$XT_WITH_ATTRIBUTES" ]; then fi fi -if [ -n "$XT_WITH_ASSERTIONS" ]; then +if [[ -n "$XT_WITH_ASSERTIONS" ]]; then assertions="$XT_WITH_ASSERTIONS" - if [ -f "$assertions" ]; then + if [[ -f "$assertions" ]]; then assertions=$(realpath "$assertions") echo "Assertions are a file: $assertions" args+=(--assertions "$assertions") - elif [ "$(echo "$assertions" | jq -e . >/dev/null 2>&1 && echo valid || echo invalid)" == "valid" ]; then + elif [[ "$(echo "$assertions" | jq -e . >/dev/null 2>&1 && echo valid || echo invalid)" == "valid" ]]; then # Assertions are plain json echo "Assertions are plain json: $assertions" args+=(--assertions "$assertions") @@ -168,9 +187,9 @@ if [ -n "$XT_WITH_ASSERTIONS" ]; then fi fi -if [ -n "$XT_WITH_ASSERTION_VERIFICATION_KEYS" ]; then +if [[ -n "$XT_WITH_ASSERTION_VERIFICATION_KEYS" ]]; then verification_keys="$XT_WITH_ASSERTION_VERIFICATION_KEYS" - if [ -f "$verification_keys" ]; then + if [[ -f "$verification_keys" ]]; then verification_keys=$(realpath "$verification_keys") echo "Verification keys are a file: $verification_keys" args+=(--assertionVerificationKeys "$verification_keys") @@ -185,39 +204,56 @@ if ! cd "$SCRIPT_DIR"; then exit 1 fi -if [ "$1" == "encrypt" ]; then +# Echo a CLI invocation with the --auth secret masked, so CI logs never capture +# client credentials. The real (unmasked) args are still used for execution. +echo_redacted() { + local out=() a mask_next=0 + for a in "$@"; do + if [[ "$mask_next" == 1 ]]; then + out+=("${a%%:*}:***") + mask_next=0 + elif [[ "$a" == "--auth" ]]; then + out+=("$a") + mask_next=1 + else + out+=("$a") + fi + done + echo "${out[@]}" + return 0 +} + +if [[ "$1" == "encrypt" ]]; then if npx $CTL help | grep autoconfigure; then args+=(--policyEndpoint "$PLATFORMURL" --autoconfigure true) fi - if [ -n "$XT_WITH_ECDSA_BINDING" ]; then - if [ "$XT_WITH_ECDSA_BINDING" == "true" ]; then - args+=(--policyBinding ecdsa) - fi + if [[ "$XT_WITH_ECDSA_BINDING" == "true" ]]; then + args+=(--policyBinding ecdsa) fi - if [ "$XT_WITH_ECWRAP" == 'true' ]; then + if [[ "$XT_WITH_ECWRAP" == 'true' ]]; then args+=(--encapKeyType "ec:secp256r1") fi - if [ "$XT_WITH_PLAINTEXT_POLICY" == "true" ]; then + if [[ "$XT_WITH_PLAINTEXT_POLICY" == "true" ]]; then args+=(--policyType plaintext) fi - if [ -n "$XT_WITH_TARGET_MODE" ]; then + if [[ -n "$XT_WITH_TARGET_MODE" ]]; then args+=(--tdfSpecVersion "$XT_WITH_TARGET_MODE") fi - echo npx $CTL encrypt "$src_file" "${args[@]}" + echo_redacted npx $CTL encrypt "$src_file" "${args[@]}" npx $CTL encrypt "$src_file" "${args[@]}" -elif [ "$1" == "decrypt" ]; then - if [ "$XT_WITH_VERIFY_ASSERTIONS" == 'false' ]; then +elif [[ "$1" == "decrypt" ]]; then + if [[ "$XT_WITH_VERIFY_ASSERTIONS" == 'false' ]]; then args+=(--noVerifyAssertions) fi - if [ "$XT_WITH_ECWRAP" == 'true' ]; then + if [[ "$XT_WITH_ECWRAP" == 'true' ]]; then args+=(--rewrapKeyType "ec:secp256r1") fi - if [ -n "$XT_WITH_KAS_ALLOW_LIST" ]; then + if [[ -n "$XT_WITH_KAS_ALLOW_LIST" ]]; then args+=(--allowList "$XT_WITH_KAS_ALLOW_LIST") fi - if [ "$XT_WITH_IGNORE_KAS_ALLOWLIST" == "true" ]; then + if [[ "$XT_WITH_IGNORE_KAS_ALLOWLIST" == "true" ]]; then args+=(--ignoreAllowList) fi # only ignore allowlist if the kas allowlist fetching from kas registry has not been implemented @@ -227,7 +263,7 @@ elif [ "$1" == "decrypt" ]; then args+=(--ignoreAllowList) fi - echo npx $CTL decrypt "$src_file" "${args[@]}" + echo_redacted npx $CTL decrypt "$src_file" "${args[@]}" npx $CTL decrypt "$src_file" "${args[@]}" else echo "Incorrect argument provided" diff --git a/xtest/test_dpop.py b/xtest/test_dpop.py index 085f71fc..18844774 100644 --- a/xtest/test_dpop.py +++ b/xtest/test_dpop.py @@ -305,6 +305,37 @@ def _post_rewrap( ) +def _post_rewrap_with_nonce_retry( + call: RewrapCall, + key: DPoPKey, + *, + access_token: str, + auth_scheme: str = "DPoP", +) -> requests.Response: + """POST a rewrap, transparently satisfying a `require_nonce` challenge. + + Mints a fresh DPoP proof (no nonce) and sends it. If the KAS replies with a + `401` carrying a `DPoP-Nonce` header (the `use_dpop_nonce` challenge), the + proof is re-minted with that nonce and the request is retried once, keeping + the same `auth_scheme`. This makes callers agnostic to whether + `require_nonce` is enabled on the target KAS: with it off the first request + is returned as-is; with it on the second (nonce-bearing) request is. + """ + proof = key.sign_dpop_proof(htm="POST", htu=call.url, access_token=access_token) + response = _post_rewrap( + call, access_token=access_token, dpop_proof=proof, auth_scheme=auth_scheme + ) + nonce = response.headers.get("DPoP-Nonce") + if response.status_code == 401 and nonce: + proof = key.sign_dpop_proof( + htm="POST", htu=call.url, access_token=access_token, nonce=nonce + ) + response = _post_rewrap( + call, access_token=access_token, dpop_proof=proof, auth_scheme=auth_scheme + ) + return response + + def _assert_unauthorized(response: requests.Response) -> None: assert response.status_code == 401, response.text # Confirm the rejection is actually a DPoP-related challenge so a 401 @@ -323,9 +354,11 @@ def _skip_unless_dpop_enabled(encrypt_sdk: tdfs.SDK, in_focus: set[tdfs.SDK]) -> @pytest.fixture(autouse=True) def _dpop_client_env(monkeypatch: pytest.MonkeyPatch) -> None: - # SDK CLI shims read CLIENTID from the environment; tests in this module - # must use the DPoP-bound client provisioned by `service provision keycloak`. + # SDK CLI shims read CLIENTID/XT_WITH_DPOP from the environment; tests in + # this module must use the DPoP-bound client provisioned by + # `service provision keycloak` and enable DPoP proof generation. monkeypatch.setenv("CLIENTID", "opentdf-dpop") + monkeypatch.setenv("XT_WITH_DPOP", "ES256") def test_dpop_happy_path_roundtrip( @@ -426,16 +459,14 @@ def test_dpop_bearer_scheme_warns_but_accepted_for_dpop_token( dpop_access = _get_dpop_access_token() rewrap_call = _signed_rewrap_request(ct_file, dpop_access.key) - bearer_proof = dpop_access.key.sign_dpop_proof( - htm="POST", - htu=rewrap_call.url, - access_token=dpop_access.token, - ) + # Both calls go through the nonce-retry helper so the test passes whether or + # not the target KAS has `require_nonce` enabled: when it is, the lenient + # accept (and the WARN) only happen after the nonce challenge is satisfied. mark = audit_logs.mark("before_bearer_scheme_request") - bearer_response = _post_rewrap( + bearer_response = _post_rewrap_with_nonce_retry( rewrap_call, + dpop_access.key, access_token=dpop_access.token, - dpop_proof=bearer_proof, auth_scheme="Bearer", ) @@ -447,16 +478,10 @@ def test_dpop_bearer_scheme_warns_but_accepted_for_dpop_token( ) # Compliant path control: same proof key, same token, just the right scheme. - # Distinct jti via fresh proof so the server's replay cache doesn't reject it. - dpop_proof = dpop_access.key.sign_dpop_proof( - htm="POST", - htu=rewrap_call.url, - access_token=dpop_access.token, - ) - dpop_response = _post_rewrap( + dpop_response = _post_rewrap_with_nonce_retry( rewrap_call, + dpop_access.key, access_token=dpop_access.token, - dpop_proof=dpop_proof, auth_scheme="DPoP", ) assert dpop_response.status_code == 200, dpop_response.text