Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7c5f31a

Browse files
ThomasK33ThomasK33
authored
🤖 refactor: add bounded history scalar evidence (#4215)
Adds bounded scalar facts and canonical JSON evidence above #4214's raw token reader. This is an inactive prerequisite for the oversized-history witness fix in #4182; it does not decide message readability or grant cancellation authority. Strings retain a bounded prefix, length, digest and optional exact-match result. Numbers retain up to 2,048 significant digits plus exponent/position and a nonzero-tail flag, then delegate rounding to native Number. The retained prefix exceeds every finite binary64 rounding boundary's decimal expansion; a discarded nonzero tail preserves the side of that boundary. Canonical evidence accounts for escaping, split surrogate pairs, integer-property ordering, duplicates, raw digest/length and caller-reported normalization changes. Validation: reader/scalar tests previously passed 91 cases with 4,712 assertions; the final reader/scalar/projection combination passes 109 tests / 7,339 assertions. A Node 22 native oracle passes 4,704 checks, including midpoint/subnormal/overflow edges, huge exponent cancellation, 2,000 generated numbers and an 8 MiB canonical row. Both owned layers passed canonical static checks and independent review before composition; production scalar code remained unchanged during final test lint fixes. Risk: incorrect number or canonical equivalence could misclassify later history evidence, so this module stays inactive until the history-specific adapter is reviewed. Retention is bounded for scalars; nesting/key bookkeeping follows the raw reader's documented limits. No new persistence format or schema interpreter. The numeric proof comment names SIGNIFICANT_DIGITS so its retention bound cannot drift from the implementation. This review correction changes no behavior. Two fork/send integration assertions retain their success requirement while exposing the returned error when a request fails. Scalar-reader production is unchanged. Integrated validation on #4191 `87badaf62f925e118c2292cf3e788fc92894dafc` and #4209 `d83e1cb55f50676562e675398f76e0b5118053ab` above main `0d31680932fe5b2cb33ce33ec43cb576c86ee0e6`: six affected suites passed on each integrated tree: A 549 tests / 3,337 assertions and B 602 tests / 3,649 assertions; full TypeScript and canonical static checks passed on both. The rebased A/B trees exactly match those validated trees. This stack integrates main `0d31680932fe5b2cb33ce33ec43cb576c86ee0e6`, including #4225’s workspace-fork recovery fix. Each layer retains its previously reviewed diff and all addressed review fixes. The complete cancellation phase is ordered #4214 → #4215 → #4219 → #4221 → #4182 → #4187 → #4191 → #4209. The reader prerequisites and runtime changes merge together only after every member has current-head approval and green CI. --- _Generated with `xum` • Model: `unavailable` • Thinking: `unavailable` • Cost: `$unavailable`_ <!-- mux-attribution: model=unavailable thinking=unavailable costs=unavailable -->
1 parent 5e6bbaf commit 7c5f31a

3 files changed

Lines changed: 490 additions & 2 deletions

File tree

‎src/node/services/historyScalarEvidence.test.ts‎

Lines changed: 244 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,244 @@
1+
import { afterEach, describe, expect, test } from "bun:test";
2+
import { mkdtemp, open, rm, writeFile } from "node:fs/promises";
3+
import { tmpdir } from "node:os";
4+
import { join } from "node:path";
5+
import { createHash } from "node:crypto";
6+
import { scanHistoryRows } from "./historyRowScanner";
7+
import {
8+
createHistoryCanonicalEvidence,
9+
createHistoryNumberEvidence,
10+
createHistoryStringEvidence,
11+
} from "./historyScalarEvidence";
12+
13+
const directories: string[] = [];
14+
afterEach(async () => {
15+
for (const directory of directories.splice(0))
16+
await rm(directory, { recursive: true, force: true });
17+
});
18+
19+
function numberEvidence(raw: string, chunkSize: number) {
20+
const evidence = createHistoryNumberEvidence(raw.length);
21+
for (let i = 0; i < raw.length; i += chunkSize) evidence.push(raw.slice(i, i + chunkSize));
22+
return evidence.finish();
23+
}
24+
25+
async function canonicalRows(raw: string | Buffer, normalizationChanged = false) {
26+
const directory = await mkdtemp(join(tmpdir(), "history-scalar-test-"));
27+
directories.push(directory);
28+
const file = join(directory, "rows.jsonl");
29+
await writeFile(file, raw);
30+
const result: boolean[] = [];
31+
await scanHistoryRows(file, () => {
32+
const evidence = createHistoryCanonicalEvidence(Buffer.byteLength(raw));
33+
return {
34+
token: evidence.token.bind(evidence),
35+
finish: (row) => {
36+
result.push(evidence.finish(row, normalizationChanged));
37+
},
38+
};
39+
});
40+
return result;
41+
}
42+
43+
function decimalDyadic(numerator: bigint, denominatorPower: number) {
44+
const digits = (numerator * 5n ** BigInt(denominatorPower))
45+
.toString()
46+
.padStart(denominatorPower + 1, "0");
47+
return `${digits.slice(0, -denominatorPower)}.${digits.slice(-denominatorPower)}`;
48+
}
49+
50+
function changeLastDecimalDigit(value: string, delta: bigint) {
51+
const [whole, fraction] = value.split(".");
52+
const digits = (BigInt(whole + fraction) + delta)
53+
.toString()
54+
.padStart(whole.length + fraction.length, "0");
55+
return `${digits.slice(0, -fraction.length)}.${digits.slice(-fraction.length)}`;
56+
}
57+
58+
describe("bounded history scalar evidence", () => {
59+
test.each([1, 2, 7, 63, 4096])(
60+
"native number conversion and canonicality agree across %s-character chunks",
61+
(chunkSize) => {
62+
const values = [
63+
"0",
64+
"-0",
65+
"0.0",
66+
"-0e99999",
67+
"1",
68+
"-1",
69+
"1e0",
70+
"1E+0",
71+
"1e309",
72+
"-1e309",
73+
"1e-99999",
74+
"-1e-99999",
75+
"1.7976931348623157e308",
76+
"5e-324",
77+
"2.2250738585072014e-308",
78+
"9007199254740993",
79+
"9007199254740995",
80+
"1.00000000000000011102230246251565404236316680908203125",
81+
"0." + "0".repeat(5000) + "12345e5004",
82+
"12345" + "0".repeat(5000) + "e-5004",
83+
"1e+" + "0".repeat(5000) + "308",
84+
"1e-" + "9".repeat(5000),
85+
"1e" + "9".repeat(5000),
86+
"1." + "0".repeat(7000) + "1",
87+
];
88+
for (const raw of values) {
89+
const actual = numberEvidence(raw, chunkSize);
90+
const expected: unknown = JSON.parse(raw);
91+
expect(Object.is(actual.value, expected)).toBe(true);
92+
expect(actual.canonical).toBe(JSON.stringify(expected) === raw);
93+
expect(actual.retainedSignificantDigits).toBeLessThanOrEqual(2048);
94+
}
95+
}
96+
);
97+
98+
test("native rounding preserves midpoint ties and sticky tails beyond retained digits", () => {
99+
const midpoints = [
100+
decimalDyadic((1n << 53n) + 1n, 53),
101+
decimalDyadic((1n << 53n) + 3n, 53),
102+
decimalDyadic(1n, 1075),
103+
decimalDyadic(3n, 1075),
104+
decimalDyadic((1n << 53n) - 1n, 1075),
105+
];
106+
for (const midpoint of midpoints) {
107+
const tie = midpoint + "0".repeat(3000);
108+
for (const raw of [tie, changeLastDecimalDigit(tie, -1n), changeLastDecimalDigit(tie, 1n)]) {
109+
for (const sign of ["", "-"]) {
110+
const literal = sign + raw;
111+
expect(Object.is(numberEvidence(literal, 13).value, JSON.parse(literal))).toBe(true);
112+
}
113+
}
114+
}
115+
const overflow = ((1n << 1024n) - (1n << 970n)).toString() + "." + "0".repeat(3000);
116+
for (const raw of [
117+
overflow,
118+
changeLastDecimalDigit(overflow, -1n),
119+
changeLastDecimalDigit(overflow, 1n),
120+
])
121+
expect(Object.is(numberEvidence(raw, 19).value, JSON.parse(raw))).toBe(true);
122+
});
123+
124+
test("many generated finite numbers and noncanonical spellings use native conversion", () => {
125+
let seed = 123456789;
126+
for (let i = 0; i < 2000; i++) {
127+
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
128+
const value = (seed - 0x80000000) * 10 ** ((i % 600) - 300);
129+
const raw = JSON.stringify(value);
130+
const expected: unknown = JSON.parse(raw);
131+
expect(Object.is(numberEvidence(raw, 3).value, expected)).toBe(true);
132+
expect(numberEvidence(raw, 3).canonical).toBe(true);
133+
}
134+
});
135+
136+
test("giant numeric and string inputs retain only bounded prefixes", () => {
137+
const number = createHistoryNumberEvidence(16 * 1024 * 1024);
138+
number.push("1.");
139+
const zeros = "0".repeat(8192);
140+
for (let i = 0; i < 1024; i++) number.push(zeros);
141+
number.push("1");
142+
expect(number.finish()).toMatchObject({
143+
value: 1,
144+
canonical: false,
145+
retainedSignificantDigits: 2048,
146+
});
147+
const string = createHistoryStringEvidence(128, "different");
148+
for (let i = 0; i < 1024; i++) string.push(zeros);
149+
const result = string.finish();
150+
expect(result.prefix).toHaveLength(128);
151+
expect(result.length).toBe(8 * 1024 * 1024);
152+
expect(result.matchesExpected).toBe(false);
153+
});
154+
155+
test("decoded string equality and hashing preserve split surrogate pairs and lone surrogates", () => {
156+
const value = "a😀\ud800b\udfff";
157+
const probe = createHistoryStringEvidence(2, value);
158+
for (const unit of value.split("")) probe.push(unit);
159+
const actual = probe.finish();
160+
expect(actual.matchesExpected).toBe(true);
161+
expect(actual.prefix).toBe(value.slice(0, 2));
162+
expect(actual.sha256).toBe(
163+
createHash("sha256").update(Buffer.from(value, "utf16le")).digest("hex")
164+
);
165+
const other = createHistoryStringEvidence(0, value);
166+
other.push(value.replace("\ud800", "\ud801"));
167+
expect(other.finish().matchesExpected).toBe(false);
168+
});
169+
170+
test("source limits are enforced rather than silently saturating an undersized promise", () => {
171+
expect(() => createHistoryNumberEvidence(-1)).toThrow();
172+
expect(() => createHistoryNumberEvidence(1).push("10")).toThrow();
173+
expect(() => createHistoryStringEvidence(-1)).toThrow();
174+
});
175+
});
176+
177+
describe("streamed canonical JSON evidence", () => {
178+
test.each([
179+
"null",
180+
"true",
181+
"false",
182+
"0",
183+
"-0",
184+
"1e0",
185+
"1e400",
186+
'""',
187+
'"hello"',
188+
"[]",
189+
"{}",
190+
'[1,{"text":"hello"},null]',
191+
'{"2":0,"10":0,"name":0}',
192+
'{"10":0,"2":0}',
193+
'{"name":0,"2":0}',
194+
'{"01":0,"1":0}',
195+
'{"1":0,"01":0}',
196+
'{"4294967294":0,"4294967295":0}',
197+
'{"4294967295":0,"4294967294":0}',
198+
'{"a":0,"a":1}',
199+
'{"a":0,"\\u0061":1}',
200+
'{"nested":{"b":1,"b":2}}',
201+
'{ "a": 1 }',
202+
'"\\u0061"',
203+
'"a/b"',
204+
'"a\\/b"',
205+
'"\\ud800"',
206+
'"\\ud800\\udc00"',
207+
JSON.stringify("😀\ud800\n\t\u2028\u2029"),
208+
'{"__proto__":{"a":1},"constructor":null}',
209+
])("matches native JSON.stringify round trip for %s", async (raw) => {
210+
const expected = JSON.stringify(JSON.parse(raw)) === raw;
211+
expect(await canonicalRows(raw)).toEqual([expected]);
212+
expect(await canonicalRows(raw + "\n")).toEqual([expected]);
213+
});
214+
215+
test("invalid UTF-8, malformed JSON and a declared legacy normalization never prove canonical bytes", async () => {
216+
expect(await canonicalRows(Buffer.from([0x22, 0xff, 0x22]))).toEqual([false]);
217+
expect(await canonicalRows('{"unfinished":\n{"valid":true}\n')).toEqual([false, true]);
218+
expect(await canonicalRows('{"metadata":{"cmuxMetadata":{}}}', true)).toEqual([false]);
219+
});
220+
221+
test("giant canonical scalars cross raw read and Unicode boundaries without row assembly", async () => {
222+
const directory = await mkdtemp(join(tmpdir(), "history-canonical-giant-"));
223+
directories.push(directory);
224+
const file = join(directory, "row.jsonl");
225+
const chunk = "x".repeat(65529) + "😀\n";
226+
await using handle = await open(file, "w");
227+
await handle.write('{"text":"');
228+
for (let i = 0; i < 129; i++) await handle.write(JSON.stringify(chunk).slice(1, -1));
229+
await handle.write('","tail":"\\ud800"}\n');
230+
const size = (await handle.stat()).size;
231+
const results: boolean[] = [];
232+
await scanHistoryRows(file, () => {
233+
const evidence = createHistoryCanonicalEvidence(size);
234+
return {
235+
token: evidence.token.bind(evidence),
236+
finish: (row) => {
237+
results.push(evidence.finish(row));
238+
},
239+
};
240+
});
241+
expect(size).toBeGreaterThan(8 * 1024 * 1024);
242+
expect(results).toEqual([true]);
243+
});
244+
});

0 commit comments

Comments
 (0)