Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 0b0fd5e

Browse files
vaaraioclaude
vaaraio
and
claude
committed
test(sep2828): SEP-owned decision/outcome pairing conformance vectors
Six committed cases under tests/vectors/decision_pairing_v0 with a stdlib-only independent walker (no Vaara imports): valid pair, decision- only escalate, substituted attestation backlink, substituted pairing nonce, equal-decidedAt supersession tie (open contract), and fallback request-envelope binding with replay. Pairing follows the shipped shared-attestation model (records_paired). Requested on modelcontextprotocol/modelcontextprotocol#2828. Co-Authored-By: Claude Opus 4.8 <[email protected]>
1 parent d899c6c commit 0b0fd5e

26 files changed

Lines changed: 866 additions & 0 deletions

File tree

‎scripts/generate_decision_pairing_vectors.py‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
#!/usr/bin/env python3
2+
"""Generate the SEP-2828 decision/outcome pairing conformance vectors.
3+
4+
Writes signed fixtures under ``tests/vectors/decision_pairing_v0/``. Each
5+
case holds the attestation, decision record, optional execution receipt,
6+
and an ``expected.json`` of verdicts an independent verifier MUST
7+
reproduce from the committed wire bytes alone. These are the SEP-owned
8+
fixtures requested on modelcontextprotocol/modelcontextprotocol#2828.
9+
10+
Pairing model is the one the reference impl ships: a decision and a
11+
receipt pair when both carry the same attestation back-link
12+
(``records_paired``: attestationDigest constant-time equal AND
13+
attestationNonce equal). The "outcome resolves to the decision content
14+
digest" join discussed in #2828 is a distinct contract and is NOT
15+
exercised here; adopting it normatively adds a field and a case.
16+
17+
Usage: python scripts/generate_decision_pairing_vectors.py
18+
"""
19+
20+
from __future__ import annotations
21+
22+
import hashlib
23+
import json
24+
from pathlib import Path
25+
26+
from cryptography.hazmat.primitives import serialization
27+
from cryptography.hazmat.primitives.asymmetric import ec
28+
29+
from vaara.attestation._sep2787_canonical import canonical_json
30+
from vaara.attestation.decision import (
31+
DecisionDerived, emit_decision_record, make_back_link,
32+
)
33+
from vaara.attestation.receipt import (
34+
BackLink, OutcomeDerived, emit_receipt, make_result_digest,
35+
)
36+
from vaara.attestation.sep2787 import (
37+
PayloadDerived, PlannerDeclared, ToolCallBinding,
38+
emit_attestation, make_args_digest,
39+
)
40+
41+
ROOT = Path(__file__).resolve().parent.parent
42+
OUT = ROOT / "tests" / "vectors" / "decision_pairing_v0"
43+
HS = bytes.fromhex("42" * 32)
44+
IAT = "2026-06-01T10:00:00Z"
45+
RESULT = {"rows": 10, "table": "employees"}
46+
ARGS = {"table": "employees", "limit": 10}
47+
COMMON = dict(iss="issuer://test", sub="agent:reader",
48+
secret_version="v1", iat=IAT)
49+
50+
51+
def _write(path: Path, obj: dict) -> None:
52+
path.parent.mkdir(parents=True, exist_ok=True)
53+
path.write_text(json.dumps(obj, indent=2, sort_keys=True) + "\n")
54+
55+
56+
def _attestation(nonce="fixed-attestation-nonce-000"):
57+
payload = PayloadDerived(tool_calls=(ToolCallBinding(
58+
name="query_table", server_fingerprint="sha256:" + "1" * 64,
59+
args=make_args_digest(ARGS)),))
60+
return emit_attestation(
61+
planner_declared=PlannerDeclared(intent="show 10 employees"),
62+
payload_derived=payload, iss="issuer://test", sub="agent:reader",
63+
secret_version="v1", alg="HS256", signing_material=HS,
64+
nonce=nonce, iat=IAT)
65+
66+
67+
def _decision(bl, verdict, *, key, alg, nonce):
68+
return emit_decision_record(
69+
back_link=bl, decision_derived=DecisionDerived(
70+
decision=verdict, decided_at=IAT, risk_score="0.21",
71+
threshold_allow="0.30", threshold_block="0.80",
72+
policy_id="policy:read-only/3"),
73+
alg=alg, signing_material=key, nonce=nonce, **COMMON)
74+
75+
76+
def _receipt(bl, status, *, key, alg, nonce, commit=True):
77+
return emit_receipt(
78+
back_link=bl, outcome_derived=OutcomeDerived(
79+
status=status, completed_at=IAT,
80+
result_commitment=make_result_digest(RESULT) if commit else None),
81+
alg=alg, signing_material=key, nonce=nonce, **COMMON)
82+
83+
84+
def _fallback_backlink() -> BackLink:
85+
# No SEP-2787 attestation: bind to SHA-256 over the JCS-canonical
86+
# observed request envelope plus a server nonce. The BackLink is an
87+
# opaque sha256: digest + nonce, so the fallback is what the
88+
# enforcement point commits to as the digest.
89+
envelope = {"method": "tools/call", "params": {
90+
"name": "query_table", "arguments": ARGS,
91+
"_meta": {"io.modelcontextprotocol/aiInvocation": {"turnId": "turn-7"}}}}
92+
digest = hashlib.sha256(canonical_json(envelope)).hexdigest()
93+
return BackLink(attestation_digest="sha256:" + digest,
94+
attestation_nonce="server-chosen-nonce-001")
95+
96+
97+
def main() -> None:
98+
es = ec.generate_private_key(ec.SECP256R1())
99+
keys = OUT / "keys"
100+
keys.mkdir(parents=True, exist_ok=True)
101+
(keys / "hs256_secret.bin").write_bytes(HS)
102+
(keys / "es256_private.pem").write_bytes(es.private_bytes(
103+
serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8,
104+
serialization.NoEncryption()))
105+
(keys / "es256_public.pem").write_bytes(es.public_key().public_bytes(
106+
serialization.Encoding.PEM,
107+
serialization.PublicFormat.SubjectPublicKeyInfo))
108+
109+
att = _attestation()
110+
bl = make_back_link(att)
111+
112+
def case(name, *, decision, receipt=None, expected):
113+
d = OUT / "normative" / name
114+
_write(d / "attestation.json", att.to_dict())
115+
_write(d / "decision.json", decision.to_dict())
116+
if receipt is not None:
117+
_write(d / "receipt.json", receipt.to_dict())
118+
_write(d / "expected.json", expected)
119+
120+
# 1. Valid allow + executed, paired on the same attestation back-link.
121+
case("valid_pair_allow_executed",
122+
decision=_decision(bl, "allow", key=HS, alg="HS256", nonce="d1"),
123+
receipt=_receipt(bl, "executed", key=HS, alg="HS256", nonce="r1"),
124+
expected={"decision_signature_ok": True, "decision_back_link_ok": True,
125+
"receipt_signature_ok": True, "receipt_back_link_ok": True,
126+
"records_paired": True})
127+
128+
# 2. Decision-only escalate: valid, no sibling outcome, not an error.
129+
case("decision_only_escalate",
130+
decision=_decision(bl, "escalate", key=es, alg="ES256", nonce="d2"),
131+
expected={"decision_signature_ok": True, "decision_back_link_ok": True,
132+
"receipt_present": False, "outcome_required": False})
133+
134+
# 3. Substituted attestation back-link: receipt binds a different
135+
# attestation. Both verify alone; they do not pair.
136+
other = _attestation(nonce="other-attestation-nonce-999")
137+
case("substituted_attestation_backlink",
138+
decision=_decision(bl, "allow", key=HS, alg="HS256", nonce="d3"),
139+
receipt=_receipt(make_back_link(other), "executed", key=HS,
140+
alg="HS256", nonce="r3"),
141+
expected={"decision_signature_ok": True, "decision_back_link_ok": True,
142+
"receipt_signature_ok": True,
143+
"receipt_back_link_ok_against_stored_attestation": False,
144+
"records_paired": False})
145+
146+
# 4. Substituted pairing nonce: same digest, mismatched nonce. The
147+
# nonce is the pairing link, so the pair is rejected.
148+
tampered = BackLink(attestation_digest=bl.attestation_digest,
149+
attestation_nonce="substituted-nonce")
150+
case("substituted_pairing_nonce",
151+
decision=_decision(bl, "allow", key=HS, alg="HS256", nonce="d4"),
152+
receipt=_receipt(tampered, "executed", key=HS, alg="HS256", nonce="r4"),
153+
expected={"records_paired": False,
154+
"note": "same attestationDigest, mismatched attestationNonce"})
155+
156+
# 5. Equal-decidedAt supersession tie. The reference impl has NO
157+
# supersession ordering today; this pins the OPEN contract question.
158+
d5 = OUT / "normative" / "supersession_equal_decidedat_tie"
159+
_write(d5 / "attestation.json", att.to_dict())
160+
_write(d5 / "decision_a.json",
161+
_decision(bl, "block", key=HS, alg="HS256", nonce="d5a").to_dict())
162+
_write(d5 / "decision_b.json",
163+
_decision(bl, "allow", key=HS, alg="HS256", nonce="d5b").to_dict())
164+
_write(d5 / "expected.json", {
165+
"both_signatures_ok": True, "both_back_links_ok": True, "winner": None,
166+
"open_contract": "equal decidedAt needs a deterministic tie-break "
167+
"(e.g. lexicographic on record nonce); unspecified "
168+
"in the reference impl as of v0.50.0"})
169+
170+
# 6. Fallback request-envelope binding, replay/substitution.
171+
fb = _fallback_backlink()
172+
d6 = OUT / "normative" / "fallback_envelope_binding"
173+
_write(d6 / "decision.json",
174+
_decision(fb, "allow", key=HS, alg="HS256", nonce="d6").to_dict())
175+
rec = _receipt(fb, "executed", key=HS, alg="HS256", nonce="r6")
176+
_write(d6 / "receipt.json", rec.to_dict())
177+
replayed = rec.to_dict()
178+
replayed["backLink"]["attestationDigest"] = "sha256:" + "0" * 64
179+
_write(d6 / "receipt_replayed.json", replayed)
180+
_write(d6 / "expected.json", {
181+
"decision_signature_ok": True, "receipt_signature_ok": True,
182+
"records_paired": True, "replayed_receipt_signature_ok": False,
183+
"note": "fallback binding when no SEP-2787 attestation exists; the "
184+
"BackLink digest is over the JCS-canonical request envelope"})
185+
186+
print(f"wrote vectors under {OUT}")
187+
188+
189+
if __name__ == "__main__":
190+
main()

‎tests/test_decision_pairing_vectors.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
"""Guard: the committed SEP-2828 decision/outcome vectors verify independently.
2+
3+
Runs ``_check_independent.py`` (stdlib + cryptography + rfc8785, no Vaara
4+
imports) over the committed fixtures. If the format or a fixture drifts,
5+
the second-implementation checker fails here in CI rather than silently
6+
shipping broken vectors.
7+
"""
8+
9+
from __future__ import annotations
10+
11+
import importlib.util
12+
from pathlib import Path
13+
14+
import pytest
15+
16+
for _mod in ("rfc8785", "cryptography"):
17+
if importlib.util.find_spec(_mod) is None:
18+
pytest.skip(
19+
"attestation extra not installed (pip install 'vaara[attestation]')",
20+
allow_module_level=True,
21+
)
22+
23+
VECTORS = Path(__file__).parent / "vectors" / "decision_pairing_v0"
24+
CHECKER = VECTORS / "_check_independent.py"
25+
26+
27+
def _load_checker():
28+
spec = importlib.util.spec_from_file_location(
29+
"_decision_pairing_vector_checker", CHECKER)
30+
module = importlib.util.module_from_spec(spec)
31+
spec.loader.exec_module(module)
32+
return module
33+
34+
35+
def test_independent_walker_passes_all_cases():
36+
assert _load_checker().main() == 0
37+
38+
39+
def test_at_least_six_cases_present():
40+
cases = [p for p in (VECTORS / "normative").iterdir() if p.is_dir()]
41+
assert len(cases) >= 6

‎tests/vectors/decision_pairing_v0/_check_independent.py‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
#!/usr/bin/env python3
2+
"""Independent conformance checker for the SEP-2828 decision/outcome vectors.
3+
4+
Imports only the standard library plus ``cryptography`` and ``rfc8785``.
5+
It does not import Vaara. It reads the committed fixtures from disk and
6+
reproduces signature verification, attestation back-link verification,
7+
and decision/outcome pairing for each case, then compares against
8+
``expected.json``.
9+
10+
A second implementation that can run this file (or reproduce its logic)
11+
demonstrates the decision/outcome record format is consumable without
12+
depending on Vaara. Run:
13+
``python tests/vectors/decision_pairing_v0/_check_independent.py``.
14+
Exit code 0 means every case matched its expected verdict.
15+
"""
16+
17+
from __future__ import annotations
18+
19+
import hashlib
20+
import hmac
21+
import json
22+
import sys
23+
from pathlib import Path
24+
25+
import rfc8785
26+
from cryptography.exceptions import InvalidSignature
27+
from cryptography.hazmat.primitives import hashes, serialization
28+
from cryptography.hazmat.primitives.asymmetric import ec
29+
from cryptography.hazmat.primitives.asymmetric.utils import encode_dss_signature
30+
31+
HERE = Path(__file__).resolve().parent
32+
KEYS = HERE / "keys"
33+
34+
_DECISION_BLOCKS = ("version", "alg", "backLink", "decisionDerived",
35+
"issuerAsserted")
36+
_RECEIPT_BLOCKS = ("version", "alg", "backLink", "outcomeDerived",
37+
"receiptAsserted")
38+
39+
40+
def _jcs(obj) -> bytes:
41+
return rfc8785.dumps(obj)
42+
43+
44+
def _sha256_hex(content: bytes) -> str:
45+
return f"sha256:{hashlib.sha256(content).hexdigest()}"
46+
47+
48+
def _payload(record: dict, blocks) -> bytes:
49+
return _jcs({k: record[k] for k in blocks})
50+
51+
52+
def verify_signature(record: dict, blocks) -> bool:
53+
payload = _payload(record, blocks)
54+
alg, sig = record["alg"], record["signature"]
55+
if alg == "HS256":
56+
secret = (KEYS / "hs256_secret.bin").read_bytes()
57+
expected = hmac.new(secret, payload, hashlib.sha256).hexdigest()
58+
return hmac.compare_digest(expected, sig)
59+
if alg == "ES256":
60+
pub = serialization.load_pem_public_key(
61+
(KEYS / "es256_public.pem").read_bytes())
62+
if len(sig) != 128:
63+
return False
64+
raw = bytes.fromhex(sig)
65+
der = encode_dss_signature(
66+
int.from_bytes(raw[:32], "big"), int.from_bytes(raw[32:], "big"))
67+
try:
68+
pub.verify(der, payload, ec.ECDSA(hashes.SHA256()))
69+
return True
70+
except InvalidSignature:
71+
return False
72+
return False
73+
74+
75+
def verify_back_link(record: dict, attestation: dict) -> bool:
76+
expected = _sha256_hex(_jcs(attestation))
77+
bl = record["backLink"]
78+
if not hmac.compare_digest(bl["attestationDigest"], expected):
79+
return False
80+
return bl["attestationNonce"] == attestation["issuerAsserted"]["nonce"]
81+
82+
83+
def records_paired(decision: dict, receipt: dict) -> bool:
84+
db, rb = decision["backLink"], receipt["backLink"]
85+
if not hmac.compare_digest(db["attestationDigest"], rb["attestationDigest"]):
86+
return False
87+
return db["attestationNonce"] == rb["attestationNonce"]
88+
89+
90+
def _load(case: Path, name: str):
91+
p = case / name
92+
return json.loads(p.read_text()) if p.exists() else None
93+
94+
95+
# Declarative keys carried in expected.json that are documentation, not
96+
# crypto verdicts; the checker passes them through verbatim.
97+
_DOC_KEYS = {"outcome_required", "winner", "open_contract", "note"}
98+
99+
100+
def _verdicts(case: Path, expected: dict) -> dict:
101+
att = _load(case, "attestation.json")
102+
dec = _load(case, "decision.json")
103+
rec = _load(case, "receipt.json")
104+
got: dict = {}
105+
for key in expected:
106+
if key in _DOC_KEYS:
107+
got[key] = expected[key]
108+
elif key == "decision_signature_ok":
109+
got[key] = verify_signature(dec, _DECISION_BLOCKS)
110+
elif key == "decision_back_link_ok":
111+
got[key] = verify_back_link(dec, att)
112+
elif key == "receipt_signature_ok":
113+
got[key] = verify_signature(rec, _RECEIPT_BLOCKS)
114+
elif key in ("receipt_back_link_ok",
115+
"receipt_back_link_ok_against_stored_attestation"):
116+
got[key] = verify_back_link(rec, att)
117+
elif key == "records_paired":
118+
got[key] = records_paired(dec, rec)
119+
elif key == "receipt_present":
120+
got[key] = rec is not None
121+
elif key == "both_signatures_ok":
122+
got[key] = (verify_signature(_load(case, "decision_a.json"),
123+
_DECISION_BLOCKS)
124+
and verify_signature(_load(case, "decision_b.json"),
125+
_DECISION_BLOCKS))
126+
elif key == "both_back_links_ok":
127+
got[key] = (verify_back_link(_load(case, "decision_a.json"), att)
128+
and verify_back_link(_load(case, "decision_b.json"), att))
129+
elif key == "replayed_receipt_signature_ok":
130+
got[key] = verify_signature(_load(case, "receipt_replayed.json"),
131+
_RECEIPT_BLOCKS)
132+
else:
133+
raise ValueError(f"unknown expected key: {key!r}")
134+
return got
135+
136+
137+
def main() -> int:
138+
failures = 0
139+
cases = sorted((HERE / "normative").iterdir())
140+
for case in cases:
141+
if not case.is_dir():
142+
continue
143+
expected = json.loads((case / "expected.json").read_text())
144+
got = _verdicts(case, expected)
145+
ok = got == expected
146+
failures += 0 if ok else 1
147+
print(f"[{'OK' if ok else 'FAIL'}] {case.name}: {got}")
148+
print(f"\n{len(cases) - failures}/{len(cases)} cases matched expected.")
149+
return 1 if failures else 0
150+
151+
152+
if __name__ == "__main__":
153+
sys.exit(main())

0 commit comments

Comments
 (0)