diff --git a/extension/src/rules/__tests__/unicode-invisibles-strip.test.ts b/extension/src/rules/__tests__/unicode-invisibles-strip.test.ts index 4c9a56f..b43300a 100644 --- a/extension/src/rules/__tests__/unicode-invisibles-strip.test.ts +++ b/extension/src/rules/__tests__/unicode-invisibles-strip.test.ts @@ -11,9 +11,17 @@ const ZWNBSP = "\u{FEFF}"; // BOM const WORD_JOINER = "\u{2060}"; const RLO = "\u{202E}"; // bidi override const LRI = "\u{2066}"; // bidi isolate +const SOFT_HYPHEN = "\u{00AD}"; +const CGJ = "\u{034F}"; // combining grapheme joiner +const HANGUL_FILLER = "\u{3164}"; +const HANGUL_CHOSEONG_FILLER = "\u{115F}"; +const HALFWIDTH_HANGUL_FILLER = "\u{FFA0}"; +const IVS_17 = "\u{E0100}"; // first variation selector supplement code point const ZWJ = "\u{200D}"; // preserved const ZWNJ = "\u{200C}"; // preserved const LRM = "\u{200E}"; // preserved +const EMOJI_VS16 = "\u{FE0F}"; // preserved (emoji-variant selector) +const BRAILLE_BLANK = "\u{2800}"; // preserved (load-bearing in Braille) const MUTATION_THROTTLE_MS = 250; @@ -61,6 +69,60 @@ describe("unicode-invisibles-strip text nodes", () => { expect(document.body.textContent).toBe(preserved); }); + // Soft hyphen inside a word breaks `\b\b` anchors in every other + // regex-based rule. Sighted users don't see the soft hyphen except at + // line-break opportunities; the agent reading textContent always does. + // Stripping restores the word boundary so downstream pattern rules + // (prompt-injection-redact, secrets-redact, …) see the intended phrase. + it("strips soft hyphen inside a word", () => { + document.body.innerHTML = `

igno${SOFT_HYPHEN}re this

`; + unicodeInvisiblesStripRule.apply(document.body); + + expect(document.body.textContent).toBe("ignore this"); + }); + + it("strips combining grapheme joiner", () => { + document.body.innerHTML = `

dis${CGJ}regard

`; + unicodeInvisiblesStripRule.apply(document.body); + + expect(document.body.textContent).toBe("disregard"); + }); + + it("strips Hangul filler variants", () => { + document.body.innerHTML = `

a${HANGUL_FILLER}b${HANGUL_CHOSEONG_FILLER}c${HALFWIDTH_HANGUL_FILLER}d

`; + unicodeInvisiblesStripRule.apply(document.body); + + expect(document.body.textContent).toBe("abcd"); + }); + + it("strips variation-selector-supplement code points", () => { + document.body.innerHTML = `

x${IVS_17}y

`; + unicodeInvisiblesStripRule.apply(document.body); + + expect(document.body.textContent).toBe("xy"); + }); + + // The U+FE00–FE0F variation selectors are deliberately preserved: stripping + // U+FE0F would turn red-heart emoji ("❤️") into its text variant + // and change rendering for every emoji on the page. + it("preserves the emoji variation selector U+FE0F", () => { + document.body.innerHTML = `

love \u{2764}${EMOJI_VS16} you

`; + unicodeInvisiblesStripRule.apply(document.body); + + expect(document.body.textContent).toBe(`love \u{2764}${EMOJI_VS16} you`); + }); + + // Braille text routinely contains the blank pattern as a load-bearing + // character; stripping would corrupt Braille documents. + it("preserves U+2800 BRAILLE PATTERN BLANK", () => { + document.body.innerHTML = `

${BRAILLE_BLANK}braille${BRAILLE_BLANK}

`; + unicodeInvisiblesStripRule.apply(document.body); + + expect(document.body.textContent).toBe( + `${BRAILLE_BLANK}braille${BRAILLE_BLANK}`, + ); + }); + it("leaves plain text untouched", () => { document.body.innerHTML = `

hello world

`; unicodeInvisiblesStripRule.apply(document.body); diff --git a/extension/src/rules/unicode-invisibles-strip.ts b/extension/src/rules/unicode-invisibles-strip.ts index debc81a..82ddff3 100644 --- a/extension/src/rules/unicode-invisibles-strip.ts +++ b/extension/src/rules/unicode-invisibles-strip.ts @@ -11,13 +11,17 @@ // // We strip rather than annotate because there is no legitimate reason for // these code points to appear in user-facing page text on the modern web. -// Code points with real script-shaping use are preserved: +// Code points with real script-shaping or rendering use are preserved (full +// list and rationale in the regex comment block below): // // - U+200C ZWNJ — Persian/Hindi ligature control // - U+200D ZWJ — emoji ZWJ sequences (families, flags, professions) and // Indic script joining // - U+200E LRM / U+200F RLM — directional marks routinely embedded in // bidirectional text; do not reorder evident characters +// - U+FE00–U+FE0F variation selectors — emoji text/emoji style; stripping +// would change every visible emoji glyph +// - U+2800 BRAILLE PATTERN BLANK — load-bearing in Braille documents // // Removal applies to text nodes everywhere except SCRIPT/STYLE/NOSCRIPT/ // TEMPLATE (handled by `walkTextNodes`) and to every attribute value on @@ -33,16 +37,49 @@ const RULE_ID = "unicode-invisibles-strip" as const; // Code points stripped (regex written with explicit \u{} escapes so the // source file itself contains no invisible characters): +// U+00AD SOFT HYPHEN (visible only at line-break opportunities; +// in agent-read textContent it's always present and breaks +// \b\b anchors — a one-character bypass for every +// other regex-based rule in the catalog) +// U+034F COMBINING GRAPHEME JOINER (invisible; rare legit use is +// to disambiguate canonical-equivalence sequences, +// essentially never in user-facing prose) +// U+115F HANGUL CHOSEONG FILLER (invisible placeholder for +// incomplete jamo sequences) +// U+1160 HANGUL JUNGSEONG FILLER (invisible placeholder) // U+180E MONGOLIAN VOWEL SEPARATOR (deprecated, zero-width) // U+200B ZERO WIDTH SPACE // U+202A–U+202E bidi embedding/override (LRE, RLE, PDF, LRO, RLO) // U+2060 WORD JOINER // U+2061–U+2064 invisible math operators // U+2066–U+2069 bidi isolates (LRI, RLI, FSI, PDI) +// U+3164 HANGUL FILLER (invisible; known homograph carrier) // U+FEFF ZERO WIDTH NO-BREAK SPACE / BOM +// U+FFA0 HALFWIDTH HANGUL FILLER (invisible) // U+E0000–U+E007F Tags block (steganographic ASCII carrier) +// U+E0100–U+E01EF Variation Selectors Supplement (ideographic variation +// selectors; same steganographic-carrier shape as the Tags +// block — 240 codepoints, one byte each of carrier capacity) +// +// Not stripped despite being invisible / inert, because legitimate rendering +// depends on them: +// U+180B–U+180D MONGOLIAN FREE VARIATION SELECTORS (script shaping) +// U+200C ZERO WIDTH NON-JOINER (Persian/Hindi ligature control) +// U+200D ZERO WIDTH JOINER (emoji ZWJ sequences, Indic joining) +// U+200E–U+200F LRM / RLM (directional marks in bidirectional text) +// U+2800 BRAILLE PATTERN BLANK (load-bearing in Braille text) +// U+FE00–U+FE0F VARIATION SELECTORS (emoji text/emoji variant — FE0F +// turns U+2764 into the red-heart emoji glyph; stripping +// would change every visible emoji) +// U+034F (combining grapheme joiner) and U+E0100–U+E01EF (Variation +// Selectors Supplement) are combining marks — Biome's +// `noMisleadingCharacterClass` rejects them inside a character class +// because a literal base + combining pair could read as a single +// grapheme. We split them into their own alternation arm; since the +// regex is only ever used for a global `replaceAll`, the alternation +// form is semantically identical. const INVISIBLES_RE = - /[\u{180E}\u{200B}\u{202A}-\u{202E}\u{2060}-\u{2064}\u{2066}-\u{2069}\u{FEFF}\u{E0000}-\u{E007F}]/gu; + /\u{034F}|[\u{E0100}-\u{E01EF}]|[\u{00AD}\u{115F}\u{1160}\u{180E}\u{200B}\u{202A}-\u{202E}\u{2060}-\u{2064}\u{2066}-\u{2069}\u{3164}\u{FEFF}\u{FFA0}\u{E0000}-\u{E007F}]/gu; function strip(value: string): string { return value.replaceAll(INVISIBLES_RE, "");