-
Notifications
You must be signed in to change notification settings - Fork 102
Expand file tree
/
Copy pathgenerate_adversary_incidents.py
More file actions
614 lines (535 loc) · 29.4 KB
/
generate_adversary_incidents.py
File metadata and controls
614 lines (535 loc) · 29.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
#!/usr/bin/env python3
"""
AiSOC AI-vs-AI adversary scenario generator (Wave 2 — w2-aivai).
=================================================================
Takes the deterministic 200-incident benchmark and produces a parallel
"adversary-mutated" dataset that simulates what an attacker-LLM would do
to evade a keyword/regex-based defender: synonym substitution, character
obfuscation, fragmentation, and softening of tactic-betraying language.
The mutator is INTENTIONALLY DETERMINISTIC. We don't want to ship a flaky
CI suite that depends on a live model — so the "attacker LLM" is replaced
with a content-derived hash + a curated mutation grammar. The grammar is
built directly from the defender's `_TACTIC_KEYWORDS` dictionary in
`services/agents/tests/test_mitre_accuracy.py`, so every keyword the
defender knows about gets at least one evasive alternate.
Mutation intensity per incident is bucketed by `sha256(id)`:
heavy ≈ 45% — every keyword in title+description rewritten
medium ≈ 35% — only one of the expected tactics is preserved cleanly
light ≈ 20% — light leetspeak only; defender should still catch
Targeted floor for the resulting eval suite is ~0.55, demonstrating
graceful degradation under adversarial obfuscation rather than the 0.80
defender-friendly floor of `test_mitre_accuracy.py`.
Outputs `services/agents/tests/eval_data/adversary_incidents.json`. Re-runs
are byte-identical given the same input dataset and mutation grammar.
Usage:
python3 scripts/generate_adversary_incidents.py
python3 scripts/generate_adversary_incidents.py --report
python3 scripts/generate_adversary_incidents.py --out PATH
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import sys
from pathlib import Path
from typing import Any
_REPO_ROOT = Path(__file__).resolve().parent.parent
_AGENTS_ROOT = _REPO_ROOT / "services" / "agents"
sys.path.insert(0, str(_AGENTS_ROOT))
# Pull the defender's keyword catalogue directly so we never drift from it.
from tests.test_mitre_accuracy import ( # type: ignore # noqa: E402
_TACTIC_KEYWORDS,
extract_tactics_from_text,
)
_DEFAULT_INPUT = _AGENTS_ROOT / "tests" / "eval_data" / "synthetic_incidents.json"
_DEFAULT_OUTPUT = _AGENTS_ROOT / "tests" / "eval_data" / "adversary_incidents.json"
# ---------------------------------------------------------------------------
# Mutation grammar
#
# Each entry rewrites the substring on the LEFT to one of the alternates on
# the right. Alternates are intentionally chosen so they do NOT match any
# keyword in `_TACTIC_KEYWORDS` (substring-checked at module load time
# below — `_validate_grammar()` will fail loudly if a synonym would still
# trip a defender keyword for any tactic).
#
# Substring keys are matched case-insensitively. Multi-word keys are
# preserved literally; single-word keys use word-boundary matching so e.g.
# "macro" doesn't replace "macroeconomics" (not that it appears here, but
# the principle is sound).
# ---------------------------------------------------------------------------
# Each value is a deterministic alternate the attacker might use. Picked by
# `(incident_id, keyword)` hash so a single incident is internally consistent
# but the dataset as a whole exercises the grammar broadly.
_SYNONYMS: dict[str, tuple[str, ...]] = {
# TA0001 — Initial Access
"initial access": ("first foothold", "entry stage", "ingress beachhead"),
"phishing": ("ruse mail", "deceptive lure", "ph1sh1ng", "fraudulent outreach"),
"spear-phish": ("targeted lure", "directed bait", "narrow-cast bait"),
"spear": ("focused lure", "direct lure", "targeted bait"),
"watering hole": ("co-opted landing site", "doctored portal", "ambush page"),
"supply chain": ("vendor-pipeline trojan", "build-pipeline poisoning", "third-party-build poisoning"),
"valid account": ("legitimate-id reuse", "trusted identity reuse", "clean-id reuse"),
"external remote service": ("internet-reachable gateway", "outward-facing edge daemon", "edge-facing remote daemon"),
"drive-by": ("auto-load infection", "passive-landing infection", "no-click landing"),
"usb": ("rem0vable media", "thumb-stick vector", "portable storage stick"),
"oauth consent": ("3rd-party-app grant", "delegated-grant abuse", "tenant grant abuse"),
"social engineering": ("operator deception", "human-factor pretexting", "operator manipulation"),
"social-engineering": ("operator deception", "human-factor pretexting", "operator manipulation"),
"vpn login": ("corp tunnel sign-in", "remote-access portal entry", "tunnelled sign-in"),
"new geography": ("unseen origin region", "anomalous origin", "first-time origin"),
"compromised npm": ("trojanized js-libs release", "doctored js-libs build", "polluted js-libs feed"),
"container image": ("workload bundle", "runtime workload artifact", "compute workload bundle"),
"wire transfer initial": ("ach instruction", "outgoing remittance", "treasury debit instruction"),
# TA0002 — Execution
"execution": ("payload delivery step", "binary launch", "code activation"),
"powershell": ("p0wer$hell", "p\u200bow\u200bersh\u200bell", "PoSh"),
"cmd.exe": ("command interpreter binary", "cm\u200bd.exe", "shell-host.exe"),
"script": ("scr1pt", "automation snippet", "in-line code blob"),
"macro": ("m@cro", "off1ce-automation routine", "embedded automation routine"),
"python": ("p\u200bython", "py interpreter", "py-runtime"),
"node": ("n0de runtime", "n\u200bode.js", "j$-runtime daemon"),
"certutil": ("cert\u200butil", "windows cert tool", "cert-handling lolbin"),
"wmi": ("management infra channel", "WM\u200bI", "windows-mgmt-fabric"),
"wmic": ("wm\u200bic", "mgmt-fabric cli", "windows-mgmt-cli"),
"mshta": ("ms\u200bhta", "mshta lolbin alt", "trusted-host runner"),
"process hollow": ("memory-cavity injection", "image substitution into target proc", "in-place process swap"),
"process create": ("spawn syscall", "child-proc launch", "new-proc materialization"),
"npm": ("n\u200bpm", "j$-pkg manager", "js dependency manager"),
"post-install": ("install-time hook", "hook stage after pkg add", "pkg-install hook"),
"container with": ("workload containing", "image carrying", "bundle containing"),
"docker run": ("orchestrator launch", "runc launch", "containerd run-cmd"),
"javascript": ("j$ runtime payload", "es-module payload", "browser-side code"),
"stage-2 payload": ("second-tier dropper", "follow-on artifact", "next-tier blob"),
"vba": ("v\u200bba", "off1ce-automation language", "doc-embedded autom"),
"browser exploitation": ("client-side rendering abuse", "html-engine abuse", "DOM-engine abuse"),
# TA0003 — Persistence
"persistence": ("re-establish footing", "reboot survival", "long-term foothold"),
"registry run": ("hkcu autostart key", "winreg autorun stanza", "reg-autostart entry"),
"scheduled task": ("schtasks job", "taskscheduler entry", "win-periodic job"),
"startup": ("st@rtup", "boot-time launch slot", "boot autoload"),
"service": ("d@emon entry", "long-running unit", "background unit"),
"implant": ("imp1ant", "long-residence agent", "resident dropper"),
"backdoor": ("b@ckdoor", "covert callback link", "covert reverse access"),
"rootkit": ("r00tkit", "kernel-resident hider", "stealth kernel module"),
"firmware": ("f1rmware", "low-level board code", "below-os layer"),
"uefi": ("ue\u200bfi", "early-boot f1rmware", "platform pre-boot f1rmware"),
"cron": ("cr0n", "unix periodic job", "unix sched entry"),
"wmi event": ("wm\u200bi event", "mgmt-fabric stream", "management-event stream"),
"permanent wmi": ("perm wm\u200bi", "always-on mgmt stream", "indefinite mgmt stream"),
"office add-in": ("off1ce plugin slot", "productivity-suite extension", "doc-suite extension"),
"vsto": ("vs\u200bto", "off1ce add-in tooling", "off1ce plugin tooling"),
"outlook startup": ("mailer load-time slot", "messaging-app boot slot", "mail-client boot slot"),
# TA0004 — Privilege Escalation
"privilege escalation": ("rights step-up", "rights uplift", "level-up of identity"),
"escalation": ("rights step-up", "level-up", "rights uplift"),
"escalate": ("step up rights", "gain higher rights", "uplift rights"),
"container escape": ("workload sandbox break", "namespace breakout", "isolation-layer break"),
"privileged pod": ("p0werful pod", "host-shared workload", "high-cap pod"),
"bypass": ("circumvent", "go around", "side-step"),
"elevation": ("rights step-up", "rights uplift", "perm step-up"),
"sudo": ("su\u200bdo", "root-shim", "uplifted-shim"),
"uac bypass": ("ua\u200bc circumvention", "consent-prompt sidestep", "security-prompt sidestep"),
"uac": ("ua\u200bc", "consent prompt", "security prompt"),
"fodhelper": ("f0dhelper", "trusted helper binary", "feature-on-demand helper"),
"suid": ("su\u200bid", "setuser bit", "setid-perm bit"),
"metadata service": ("im\u200bds endpoint", "m\u200betadata svc", "instance-info endpoint"),
# TA0005 — Defense Evasion
"defense evasion": ("av-d0dging", "control-skirting", "telemetry-d0dging"),
"obfuscat": ("scr@mbl", "deob h@rd'n", "morph"),
"fileless": ("on-disk-less", "no-disk-write", "ram-resident-only"),
"memory-only": ("ram-resident", "in-mem-only", "no on-disk artifact"),
"memory only": ("ram resident", "in mem only", "no on-disk artifact"),
"hollowing": ("h0llowing", "memory-cavity swap", "in-place image swap"),
"side-load": ("s1de-load", "hijack legit-binary lookup chain", "search-order load t@mpering"),
"dll": ("d\u200bll", "dynamic library", "loadable code module"),
"masquerad": ("disguis", "imitat", "sp00f"),
"encode": ("base64-wrap", "byte-wrap", "transform"),
"pack": ("p@ck", "c0mpress", "wr@p"),
"log cleared": ("logs purged", "audit history erased", "telemetry erased"),
"wevtutil": ("we\u200bvtutil", "win event tool", "win-log cli"),
"journalctl": ("j0urnalctl", "systemd journal cli", "systemd log cli"),
"vacuum": ("v@cuum", "c0mpact db", "c0mpress-truncate"),
"indicator removal": ("evidence cleanup", "trace cleanup", "footprint cleanup"),
"tampering": ("altering", "modify-on-the-fly", "d0ctoring"),
"stop defender": ("disable mde-agent", "halt windows av", "kill av-agent"),
"stop crowdstrike": ("disable cs-falcon", "halt edr-agent", "kill edr-sensor"),
"stop sysmon": ("disable sysmon-agent", "halt sysmon-collector", "kill sysmon-svc"),
"secure boot": ("s\u200becure boot", "verified boot chain", "boot integrity"),
# TA0006 — Credential Access
"credential": ("cred", "auth-token", "login secret"),
"brute force": ("repeated guess", "iterative login attempt", "dictionary attack"),
"brute-force": ("repeated-guess", "iterative-login", "dict-attempt"),
"kerberoast": ("kerb-ticket abuse", "kerb-ticket roast", "kerb-ticket grind"),
"dcsync": ("dc-replica-pull abuse", "directory replica abuse", "msrpc replica-pull abuse"),
"lsass": ("l$ass", "auth subsystem proc", "session-mgmt proc"),
"minidump": ("mini-d\u200bump", "process snapshot", "proc-snapshot"),
"dump": ("d\u200bump", "snapshot of proc memory", "proc-mem snapshot"),
"harvest": ("scrap", "skim", "lift"),
"password reset": ("pw rotation request", "self-serve cred change", "cred rotation request"),
"password spray": ("low-and-slow login attempt", "quiet cred probe", "spread-out login attempts"),
"credential spray": ("cred low-and-slow", "auth-token probe spread", "spread-out cred probe"),
"saml": ("s@ml", "federation token", "fed-assertion"),
"oauth": ("0auth", "tenant grant", "delegated grant"),
"ntlm": ("n\u200btlm", "ms-auth legacy", "challenge-response auth"),
"mimikatz": ("m1m1katz", "secrets extractor", "auth-secrets puller"),
"pat": ("p@t", "github-token", "ci-token"),
"personal access token": ("dev-portal token", "self-issued bearer token", "user-issued api token"),
"refresh token": ("re-fresh tok\u200ben", "long-lived auth handle", "session-extension token"),
"imds": ("im\u200bds", "instance metadata svc", "instance-info endpoint"),
"session token": ("ses\u200bsion token", "auth-cookie", "bearer for active session"),
"stolen session": ("hijacked sess1on", "co-opted active sess", "session-takeover"),
"tgs": ("tg\u200bs", "kerb svc ticket", "svc-ticket"),
# TA0007 — Discovery
"discovery": ("scout-stage", "look-around stage", "environment-mapping"),
"enumerat": ("listing-out", "walking the directory", "iterating across"),
"scan": ("sweep", "probe-pass", "fingerprint-pass"),
"account discovery": ("user-listing", "principal-listing", "identity-listing"),
"network share": ("smb mount", "remote folder", "fileshare mount"),
"recon": ("rec0n", "look-around stage", "environment-mapping"),
"replication": ("dir-sync abuse", "ad-sync abuse", "directory replica use"),
"ldap": ("l\u200bdap", "directory protocol", "x.500 query"),
"bloodhound": ("bl0odhound", "ad-graph mapper", "ad-relationship mapper"),
"smb share": ("smb mount", "remote folder mount", "fileshare mount"),
"share enum": ("mount listing", "fileshare listing", "smb-volume listing"),
"system info": ("host fingerprinting", "host-detail pull", "machine-detail pull"),
# TA0008 — Lateral Movement
"lateral movement": ("east-west pivot", "horizontal hop", "side-to-side traversal"),
"lateral": ("e-w hop", "horizontal", "side-to-side"),
"rdp": ("r\u200bdp", "remote desktop sess", "ts/mstsc sess"),
"pass-the-hash": ("hash-replay attack", "auth-hash replay", "secret-hash replay"),
"pass the hash": ("hash replay attack", "auth hash replay", "secret hash replay"),
"remote service": ("over-network svc invoke", "ms-rpc remote call", "rpc remote call"),
"pivoting": ("p1voting", "pivot-host hop", "in-network hop"),
"ssh": ("s\u200bsh", "secure shell", "openssh session"),
"smb scan": ("smb sweep", "fileshare probe", "windows-share probe"),
"wmic /node": ("wm\u200bic /machine", "mgmt-cli /remote", "mgmt-fabric cli /remote"),
"wmi remote": ("wm\u200bi remote", "mgmt-fabric remote", "remote mgmt-fabric"),
# TA0009 — Collection
"collection": ("data-gathering stage", "skim stage", "asset-pull stage"),
"data from local": ("data lifted from host", "local-host data pull", "on-host data pull"),
"screenshot": ("screen-capture", "ui snapshot", "display-grab"),
"keylog": ("keystroke-capture", "input-capture", "kb-monitor"),
"clipboard": ("paste buffer", "copy-buffer", "clip-store"),
"bulk download": ("mass pull", "batch fetch", "wholesale fetch"),
"pii": ("p\u200bii", "personal data", "user identity data"),
"customer record": ("customer row", "client account row", "subscriber record"),
"mailbox export": ("inbox extract", "mailstore export", "messaging-store export"),
"pst": ("p\u200bst", "outlook archive", "mail archive"),
"private repo": ("internal source repo", "non-public repo", "closed repo"),
"private repos": ("internal source repos", "non-public repos", "closed repos"),
# TA0010 — Exfiltration
"exfiltrat": ("ex-bound transfer", "outbound siphon", "ex-bound siphon"),
"c2 channel": ("operator channel", "comms-back tunnel", "callback channel"),
"cloud storage": ("object storage uplink", "blob-store push target", "external bucket"),
"dns tunnel": ("dns covert tunnel", "udp/53 covert track", "dns covert track"),
"ftp": ("f\u200btp", "file-xfer protocol", "legacy file-transfer"),
"upload": ("up\u200bload", "outbound transfer", "push-out"),
"egress": ("outbound flow", "ex-bound flow", "outward traffic"),
"google drive": ("g\u200bdrive", "drive-shared bucket", "consumer-cloud drive"),
"drive upload": ("drive push", "drive transfer", "drive sync-up"),
"personal drive": ("personal cloud bucket", "consumer cloud bucket", "non-corp drive"),
"transfer to": ("xfer to", "push to", "ship to"),
"s3 bucket": ("object-store bucket", "blob-store bucket", "aws object store"),
"s3://": ("aws://obj/", "object-store-uri", "blob-uri"),
"data egress": ("outbound data flow", "ex-bound data flow", "data outflow"),
# TA0011 — Command and Control
"command and control": ("operator-channel", "callback infra", "ops-channel"),
"c2": ("c\u200b2", "ops channel", "callback infra"),
"beacon": ("b\u200beacon", "callback ping", "regular check-in"),
"c&c": ("c\u200b&c", "ops channel", "callback infra"),
"dga": ("d\u200bga", "domain-gen algorithm", "algorithmic domains"),
"domain generation": ("algorithmic domain rotation", "rotating-domain algo", "domain-rotation algo"),
"dns query": ("d\u200bns query", "name-resolution", "name lookup"),
"covert channel": ("covert-comms link", "stealthy backchannel", "hidden link"),
"cobalt strike": ("c0balt strike", "crimson-team toolkit", "red-team-grade dropper"),
"https beacon": ("tls callback", "https callback", "tls heartbeat"),
"ja3": ("j@3", "tls fingerprint hash", "tls-handshake fingerprint"),
# TA0040 — Impact
"impact": ("damage stage", "harm stage", "ko stage"),
"ransom": ("ext0rt", "lockout-for-pay", "data-lockout demand"),
"encrypt": ("scr@mble", "lock-with-key", "cipher-lock"),
"wipe": ("zero-out", "shred", "scrub"),
"destroy": ("annihilate", "deletion-pass", "purge"),
"disrupt": ("d1srupt", "knock-offline", "make-unavailable"),
"defacement": ("page-graffiti", "site replacement", "homepage swap"),
"miner": ("c0in-miner", "compute-hijacking process", "cycles-stealing process"),
"cryptomine": ("c0in-mining", "compute hijack for currency", "cycles for currency"),
"xmrig": ("xm\u200brig", "monero-grade c0in-tool", "c0in-tool suite"),
"ddos": ("d\u200bdos", "flood attack", "volumetric flood"),
"syn flood": ("tcp-half-open flood", "tcp synflood", "tcp half-open swarm"),
"wire transfer": ("ach payment instruction", "outgoing remittance", "treasury debit instruction"),
"$250": ("a quarter-million-dollar", "USD250000", "two-fifty-K"),
"bec": ("b\u200bec", "exec-impersonation fraud", "boss-impersonation fraud"),
}
# Light obfuscation only — leetspeak, used for the "light" intensity bucket
# so the defender SHOULD still catch most cases there.
_LIGHT_LEET: dict[str, str] = {
"phishing": "phi$hing",
"powershell": "powershell", # untouched — light tier
"execution": "execution", # untouched — light tier
"credential": "credentia1",
"exfiltrat": "exfiltrat", # untouched — light tier
}
# ---------------------------------------------------------------------------
# Grammar validation — refuse to ship if a synonym would itself match a
# defender keyword for a tactic. This means "graceful degradation" is the
# only failure mode; we never accidentally trip a different tactic.
# ---------------------------------------------------------------------------
def _validate_grammar() -> None:
"""Make sure no synonym alternate contains a defender keyword."""
flat_keywords: list[str] = []
for kws in _TACTIC_KEYWORDS.values():
flat_keywords.extend(k.lower() for k in kws)
failures: list[str] = []
for src, alternates in _SYNONYMS.items():
for alt in alternates:
low = alt.lower()
for kw in flat_keywords:
# Same keyword self-match is fine (it's the original word
# we're rewriting), but any *other* keyword landing inside
# the alternate is a leak.
if kw == src.lower():
continue
if kw in low:
failures.append(
f"alternate '{alt}' for '{src}' still contains keyword '{kw}'"
)
if failures:
msg = "\n - ".join(failures[:20])
raise RuntimeError(
f"Adversary mutator grammar leaks defender keywords:\n - {msg}"
)
_validate_grammar()
# ---------------------------------------------------------------------------
# Mutation engine
# ---------------------------------------------------------------------------
# Ordered by descending length so multi-word keys are matched before their
# single-word substrings (e.g. "spear-phish" before "spear", "wmi event"
# before "wmi"). This is critical for correctness.
_KEY_ORDER: list[str] = sorted(_SYNONYMS.keys(), key=len, reverse=True)
# Pre-compile case-insensitive regexes per key. We escape the key directly
# (no word boundaries — many keys contain punctuation like "cmd.exe" or
# "s3://" where word boundaries fight us).
_KEY_REGEX: dict[str, re.Pattern[str]] = {
k: re.compile(re.escape(k), re.IGNORECASE) for k in _KEY_ORDER
}
def _bucket(incident_id: str) -> str:
"""Pick a mutation intensity bucket deterministically per incident.
Distribution targets:
heavy ≈ 45% — every keyword swapped for both expected tactics
medium ≈ 35% — one tactic preserved cleanly, others mutated
light ≈ 20% — light leetspeak; defender should still catch
The 45/35/20 split lands the eval near the ~0.55 graceful-degradation
floor with comfortable headroom for either side.
"""
h = int(hashlib.sha256(incident_id.encode()).hexdigest()[:8], 16)
bucket_pct = h % 100
if bucket_pct < 45:
return "heavy"
if bucket_pct < 80:
return "medium"
return "light"
def _pick_alternate(key: str, incident_id: str) -> str:
"""Pick one of the deterministic alternates for `key`."""
alts = _SYNONYMS[key]
h = int(hashlib.sha256(f"{incident_id}|{key}".encode()).hexdigest()[:8], 16)
return alts[h % len(alts)]
def _mutate_text_full(text: str, incident_id: str) -> str:
"""Replace every keyword in `text` with its deterministic alternate."""
out = text
for key in _KEY_ORDER:
pattern = _KEY_REGEX[key]
if pattern.search(out):
alt = _pick_alternate(key, incident_id)
out = pattern.sub(alt, out)
return out
def _mutate_text_partial(
text: str, incident_id: str, preserve_tactics: set[str]
) -> str:
"""Mutate keywords EXCEPT those tied to one preserved tactic.
The defender still has at least one route to detect the incident.
"""
preserve_keywords = {
kw.lower()
for tactic in preserve_tactics
for kw in _TACTIC_KEYWORDS.get(tactic, [])
}
out = text
for key in _KEY_ORDER:
if key.lower() in preserve_keywords:
continue
pattern = _KEY_REGEX[key]
if pattern.search(out):
alt = _pick_alternate(key, incident_id)
out = pattern.sub(alt, out)
return out
def _mutate_text_light(text: str) -> str:
"""Apply light leetspeak that should NOT defeat the defender.
Used as a control bucket — confirms the defender can still extract
tactics under low-grade obfuscation, so a regression on the heavy
bucket can't be masked by the light tier silently failing too.
"""
out = text
for src, dst in _LIGHT_LEET.items():
out = re.sub(re.escape(src), dst, out, flags=re.IGNORECASE)
return out
def mutate_incident(incident: dict[str, Any]) -> dict[str, Any]:
"""Return an adversarially-mutated copy of `incident`."""
incident_id = incident["id"]
bucket = _bucket(incident_id)
expected_tactics: list[str] = list(incident.get("expected_tactics", []))
if bucket == "heavy":
new_title = _mutate_text_full(incident["title"], incident_id)
new_description = _mutate_text_full(incident["description"], incident_id)
elif bucket == "medium":
if expected_tactics:
# Preserve the LAST expected tactic deterministically.
preserve_idx = (
int(hashlib.sha256(incident_id.encode()).hexdigest()[8:16], 16)
% len(expected_tactics)
)
preserve = {expected_tactics[preserve_idx]}
else:
preserve = set()
new_title = _mutate_text_partial(incident["title"], incident_id, preserve)
new_description = _mutate_text_partial(
incident["description"], incident_id, preserve
)
else:
new_title = _mutate_text_light(incident["title"])
new_description = _mutate_text_light(incident["description"])
return {
# Keep the same id space so the eval suite can correlate with the
# base dataset — that lets us do "which template lost coverage
# under adversary?" diffs.
"id": incident_id,
"template_id": incident.get("template_id"),
"template_index": incident.get("template_index"),
"adversary_intensity": bucket,
"title": new_title,
"description": new_description,
"expected_tactics": expected_tactics,
"expected_techniques": list(incident.get("expected_techniques", [])),
"severity": incident.get("severity"),
"response_class": incident.get("response_class"),
"evidence_keywords": list(incident.get("evidence_keywords", [])),
"original_title": incident["title"],
"original_description": incident["description"],
}
def mutate_dataset(incidents: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [mutate_incident(inc) for inc in incidents]
# ---------------------------------------------------------------------------
# Reporting helpers (used by --report and by the eval suite)
# ---------------------------------------------------------------------------
def adversary_coverage_report(
mutated: list[dict[str, Any]],
) -> dict[str, Any]:
"""Summarise how many incidents lost detectable tactics under mutation.
The defender's keyword extractor is run BOTH on the original and on the
mutated description so we can quantify graceful-degradation.
"""
bucket_counts = {"heavy": 0, "medium": 0, "light": 0}
correct_total = 0
lost_all_total = 0
per_tactic_lost: dict[str, int] = {}
per_template_correct: dict[str, dict[str, int]] = {}
for inc in mutated:
bucket_counts[inc["adversary_intensity"]] += 1
expected = set(inc["expected_tactics"])
text = f"{inc['title']}\n{inc['description']}"
predicted = extract_tactics_from_text(text)
overlap = predicted & expected
if overlap:
correct_total += 1
else:
lost_all_total += 1
for t in expected:
if t not in predicted:
per_tactic_lost[t] = per_tactic_lost.get(t, 0) + 1
tpl = inc.get("template_id") or "<unknown>"
slot = per_template_correct.setdefault(tpl, {"correct": 0, "total": 0})
slot["total"] += 1
if overlap:
slot["correct"] += 1
n = len(mutated)
return {
"incidents": n,
"buckets": bucket_counts,
"defender_correct": correct_total,
"defender_lost_all_tactics": lost_all_total,
"defender_accuracy": round(correct_total / n, 4) if n else 0.0,
"per_tactic_lost": dict(sorted(per_tactic_lost.items())),
"per_template": {
t: {
"correct": v["correct"],
"total": v["total"],
"accuracy": round(v["correct"] / v["total"], 4) if v["total"] else 0.0,
}
for t, v in sorted(per_template_correct.items())
},
}
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
def main() -> None:
parser = argparse.ArgumentParser(
description="Generate the adversary-mutated incident dataset (Wave 2 — w2-aivai)."
)
parser.add_argument(
"--input",
type=Path,
default=_DEFAULT_INPUT,
help="Path to base synthetic_incidents.json (default: services/agents/tests/eval_data/synthetic_incidents.json)",
)
parser.add_argument(
"--out",
type=Path,
default=_DEFAULT_OUTPUT,
help="Path to write adversary_incidents.json",
)
parser.add_argument(
"--report",
action="store_true",
help="Print a coverage / graceful-degradation report alongside writing the file.",
)
args = parser.parse_args()
if not args.input.exists():
sys.exit(
f"input not found: {args.input}. Run scripts/generate_eval_incidents.py first."
)
base = json.loads(args.input.read_text())
mutated = mutate_dataset(base)
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(mutated, indent=2) + "\n")
if args.report:
report = adversary_coverage_report(mutated)
print()
print("=" * 78)
print(" AiSOC AI-vs-AI adversary dataset — graceful-degradation report")
print("=" * 78)
print(f" Incidents: {report['incidents']}")
print(
f" Bucket split: "
f"heavy={report['buckets']['heavy']}, "
f"medium={report['buckets']['medium']}, "
f"light={report['buckets']['light']}"
)
print(
f" Defender catch rate: {report['defender_correct']}"
f"/{report['incidents']} = {report['defender_accuracy'] * 100:.1f}%"
)
print(
f" Defender lost-everything: "
f"{report['defender_lost_all_tactics']} incidents"
)
print()
print(" Tactics most-lost under mutation:")
for t, n in sorted(
report["per_tactic_lost"].items(), key=lambda kv: kv[1], reverse=True
)[:8]:
print(f" {t}: -{n}")
print("=" * 78)
print(f"wrote {len(mutated)} mutated incidents to {args.out}")
if __name__ == "__main__":
main()