diff --git a/packages/mf2-fluent/src/fluent-to-message.ts b/packages/mf2-fluent/src/fluent-to-message.ts index dfe88611..a62cdbaf 100644 --- a/packages/mf2-fluent/src/fluent-to-message.ts +++ b/packages/mf2-fluent/src/fluent-to-message.ts @@ -2,8 +2,10 @@ import * as Fluent from '@fluent/syntax'; import deepEqual from 'fast-deep-equal'; import { Expression, - FunctionAnnotation, + FunctionRef, + InputDeclaration, Literal, + LocalDeclaration, PatternMessage, SelectMessage, VariableRef, @@ -49,16 +51,21 @@ function findSelectArgs(pattern: Fluent.Pattern): SelectArg[] { return args; } -function asSelectExpression( +function asSelectorDeclaration( { selector, defaultName, keys }: SelectArg, + index: number, detectNumberSelection: boolean = true -): Expression { +): InputDeclaration | LocalDeclaration { switch (selector.type) { case 'StringLiteral': return { - type: 'expression', - arg: asValue(selector), - annotation: { type: 'function', name: 'string' } + type: 'local', + name: `_${index}`, + value: { + type: 'expression', + arg: asValue(selector), + functionRef: { type: 'function', name: 'string' } + } }; case 'VariableReference': { let name = detectNumberSelection ? 'number' : 'string'; @@ -74,15 +81,28 @@ function asSelectExpression( } } return { - type: 'expression', - arg: asValue(selector), - annotation: { type: 'function', name } + type: 'input', + name: selector.id.name, + value: { + type: 'expression', + arg: asValue(selector), + functionRef: { type: 'function', name } + } }; } } - return asExpression(selector); + const exp = asExpression(selector); + return exp.arg?.type === 'variable' + ? { + type: 'input', + name: exp.arg.name, + value: exp as Expression + } + : { type: 'local', name: `_${index}`, value: exp }; } +function asValue(exp: Fluent.VariableReference): VariableRef; +function asValue(exp: Fluent.InlineExpression): Literal | VariableRef; function asValue(exp: Fluent.InlineExpression): Literal | VariableRef { switch (exp.type) { case 'NumberLiteral': @@ -102,14 +122,14 @@ function asExpression(exp: Fluent.Expression): Expression { return { type: 'expression', arg: asValue(exp), - annotation: { type: 'function', name: 'number' } + functionRef: { type: 'function', name: 'number' } }; case 'StringLiteral': case 'VariableReference': { return { type: 'expression', arg: asValue(exp) }; } case 'FunctionReference': { - const annotation: FunctionAnnotation = { + const annotation: FunctionRef = { type: 'function', name: exp.id.name.toLowerCase() }; @@ -130,8 +150,8 @@ function asExpression(exp: Fluent.Expression): Expression { } } return args.length > 0 - ? { type: 'expression', arg: args[0], annotation } - : { type: 'expression', annotation }; + ? { type: 'expression', arg: args[0], functionRef: annotation } + : { type: 'expression', functionRef: annotation }; } case 'MessageReference': { const msgId = exp.attribute @@ -140,13 +160,13 @@ function asExpression(exp: Fluent.Expression): Expression { return { type: 'expression', arg: { type: 'literal', value: msgId }, - annotation: { type: 'function', name: 'message' } + functionRef: { type: 'function', name: 'fluent:message' } }; } case 'TermReference': { - const annotation: FunctionAnnotation = { + const annotation: FunctionRef = { type: 'function', - name: 'message' + name: 'fluent:message' }; const msgId = exp.attribute ? `-${exp.id.name}.${exp.attribute.name}` @@ -165,7 +185,7 @@ function asExpression(exp: Fluent.Expression): Expression { return { type: 'expression', arg: { type: 'literal', value: msgId }, - annotation + functionRef: annotation }; } @@ -248,7 +268,7 @@ export function fluentToMessage( keys: key.map((k, i) => k === CATCHALL ? { type: '*', value: args[i].defaultName } - : { type: 'literal', quoted: false, value: String(k) } + : { type: 'literal', value: String(k) } ), value: [] })); @@ -299,10 +319,16 @@ export function fluentToMessage( } addParts(ast, []); + const declarations = args.map((arg, index) => + asSelectorDeclaration(arg, index, detectNumberSelection) + ); return { type: 'select', - declarations: [], - selectors: args.map(arg => asSelectExpression(arg, detectNumberSelection)), + declarations, + selectors: declarations.map(decl => ({ + type: 'variable', + name: decl.name + })), variants }; } diff --git a/packages/mf2-fluent/src/fluent.test.ts b/packages/mf2-fluent/src/fluent.test.ts index eeb77429..f51e5dc8 100644 --- a/packages/mf2-fluent/src/fluent.test.ts +++ b/packages/mf2-fluent/src/fluent.test.ts @@ -715,24 +715,24 @@ describe('fluentToResourceData', () => { const msg = data.get('multi')?.get('') as SelectMessage; expect(msg.variants.map(v => v.keys)).toMatchObject([ [ - { type: 'literal', quoted: false, value: '0' }, - { type: 'literal', quoted: false, value: 'feminine' } + { type: 'literal', value: '0' }, + { type: 'literal', value: 'feminine' } ], [ - { type: 'literal', quoted: false, value: '0' }, - { type: 'literal', quoted: false, value: 'masculine' } + { type: 'literal', value: '0' }, + { type: 'literal', value: 'masculine' } ], [ - { type: 'literal', quoted: false, value: '0' }, + { type: 'literal', value: '0' }, { type: '*', value: 'neuter' } ], [ { type: '*', value: 'other' }, - { type: 'literal', quoted: false, value: 'feminine' } + { type: 'literal', value: 'feminine' } ], [ { type: '*', value: 'other' }, - { type: 'literal', quoted: false, value: 'masculine' } + { type: 'literal', value: 'masculine' } ], [ { type: '*', value: 'other' }, @@ -779,13 +779,11 @@ describe('messagetoFluent', () => { value: { type: 'expression', arg: { type: 'variable', name: 'num' }, - annotation: { type: 'function', name: 'number' } + functionRef: { type: 'function', name: 'number' } } } ], - selectors: [ - { type: 'expression', arg: { type: 'variable', name: 'local' } } - ], + selectors: [{ type: 'variable', name: 'local' }], variants: [ { keys: [{ type: '*' }], @@ -849,12 +847,12 @@ describe('messagetoFluent', () => { { type: 'expression', arg: { type: 'literal', value: 'msg' }, - annotation: { type: 'function', name: 'message' } + functionRef: { type: 'function', name: 'fluent:message' } }, { type: 'expression', arg: { type: 'variable', name: 'local' }, - annotation: { type: 'function', name: 'message' } + functionRef: { type: 'function', name: 'fluent:message' } } ] }; @@ -891,7 +889,7 @@ describe('messagetoFluent', () => { { type: 'expression', arg: { type: 'variable', name: 'input' }, - annotation: { type: 'function', name: 'message' } + functionRef: { type: 'function', name: 'fluent:message' } } ] }; diff --git a/packages/mf2-fluent/src/functions.ts b/packages/mf2-fluent/src/functions.ts index 26a793d0..f9023bf9 100644 --- a/packages/mf2-fluent/src/functions.ts +++ b/packages/mf2-fluent/src/functions.ts @@ -48,5 +48,5 @@ export function getFluentFunctions(res: FluentMessageResource) { } Object.freeze(message); - return { message }; + return { 'fluent:message': message }; } diff --git a/packages/mf2-fluent/src/message-to-fluent.ts b/packages/mf2-fluent/src/message-to-fluent.ts index 5daa4d6b..834b4c52 100644 --- a/packages/mf2-fluent/src/message-to-fluent.ts +++ b/packages/mf2-fluent/src/message-to-fluent.ts @@ -3,7 +3,7 @@ import { CatchallKey, Declaration, Expression, - FunctionAnnotation, + FunctionRef, Literal, Message, Pattern, @@ -34,7 +34,7 @@ export type FunctionMap = Record; */ export const defaultFunctionMap: FunctionMap = { datetime: 'DATETIME', - message: FluentMessageRef, + 'fluent:message': FluentMessageRef, number: 'NUMBER', plural: 'NUMBER', string: null @@ -77,7 +77,7 @@ export function messageToFluent( })); const k0 = variants[0].keys; while (k0.length > 0) { - const sel = expressionToFluent(ctx, msg.selectors[k0.length - 1]); + const sel = variableRefToFluent(ctx, msg.selectors[k0.length - 1]); let baseKeys: (Literal | CatchallKey)[] = []; let exp: Fluent.SelectExpression | undefined; for (let i = 0; i < variants.length; ++i) { @@ -161,7 +161,7 @@ function patternToFluent(ctx: MsgContext, pattern: Pattern) { function functionRefToFluent( ctx: MsgContext, arg: Fluent.InlineExpression | null, - { name, options }: FunctionAnnotation + { name, options }: FunctionRef ): Fluent.InlineExpression { const args = new Fluent.CallArguments(); if (arg) args.positional[0] = arg; @@ -236,18 +236,10 @@ function literalToFluent({ value }: Literal) { function expressionToFluent( ctx: MsgContext, - { arg, annotation }: Expression + { arg, functionRef }: Expression ): Fluent.InlineExpression { const fluentArg = arg ? valueToFluent(ctx, arg) : null; - if (annotation) { - if (annotation.type === 'function') { - return functionRefToFluent(ctx, fluentArg, annotation); - } else { - throw new Error( - `Conversion of ${annotation.type} annotation to Fluent is not supported` - ); - } - } + if (functionRef) return functionRefToFluent(ctx, fluentArg, functionRef); if (fluentArg) return fluentArg; throw new Error('Invalid empty expression'); } @@ -273,7 +265,12 @@ function variableRefToFluent( { name }: VariableRef ): Fluent.InlineExpression { const local = ctx.declarations.find(decl => decl.name === name); - return local?.value - ? expressionToFluent(ctx, local.value) - : new Fluent.VariableReference(new Fluent.Identifier(name)); + if (local?.value) { + const idx = ctx.declarations.indexOf(local); + return expressionToFluent( + { ...ctx, declarations: ctx.declarations.slice(0, idx) }, + local.value + ); + } + return new Fluent.VariableReference(new Fluent.Identifier(name)); } diff --git a/packages/mf2-icu-mf1/src/functions.ts b/packages/mf2-icu-mf1/src/functions.ts index d03191ed..eb37591c 100644 --- a/packages/mf2-icu-mf1/src/functions.ts +++ b/packages/mf2-icu-mf1/src/functions.ts @@ -180,4 +180,9 @@ function number( * * @beta */ -export const getMF1Functions = () => ({ date, duration, number, time }); +export const getMF1Functions = () => ({ + 'mf1:date': date, + 'mf1:duration': duration, + 'mf1:number': number, + 'mf1:time': time +}); diff --git a/packages/mf2-icu-mf1/src/mf1-to-message-data.ts b/packages/mf2-icu-mf1/src/mf1-to-message-data.ts index 4c6f13ff..7eb94320 100644 --- a/packages/mf2-icu-mf1/src/mf1-to-message-data.ts +++ b/packages/mf2-icu-mf1/src/mf1-to-message-data.ts @@ -1,10 +1,10 @@ import type * as AST from '@messageformat/parser'; import type { Expression, - FunctionAnnotation, + FunctionRef, + InputDeclaration, Message, Options, - VariableRef, Variant } from 'messageformat'; @@ -54,8 +54,7 @@ function findSelectArgs(tokens: AST.Token[]): SelectArg[] { function tokenToPart( token: AST.Token, - pluralArg: string | null, - pluralOffset: number | null + pluralArg: string | null ): string | Expression { switch (token.type) { case 'content': @@ -66,9 +65,9 @@ function tokenToPart( arg: { type: 'variable', name: token.arg } }; case 'function': { - const annotation: FunctionAnnotation = { + const functionRef: FunctionRef = { type: 'function', - name: token.key + name: `mf1:${token.key}` }; if (token.param && token.param.length > 0) { let value = ''; @@ -76,66 +75,56 @@ function tokenToPart( if (pt.type === 'content') value += pt.value; else throw new Error(`Unsupported param type: ${pt.type}`); } - annotation.options = new Map([['param', { type: 'literal', value }]]); + functionRef.options = new Map([['param', { type: 'literal', value }]]); } return { type: 'expression', arg: { type: 'variable', name: token.arg }, - annotation - }; - } - case 'octothorpe': { - if (!pluralArg) return '#'; - const annotation: FunctionAnnotation = { - type: 'function', - name: 'number' - }; - if (pluralOffset) { - annotation.options = new Map([ - ['pluralOffset', { type: 'literal', value: String(pluralOffset) }] - ]); - } - return { - type: 'expression', - arg: { type: 'variable', name: pluralArg }, - annotation + functionRef }; } + case 'octothorpe': + return pluralArg + ? { type: 'expression', arg: { type: 'variable', name: pluralArg } } + : '#'; /* istanbul ignore next - never happens */ default: throw new Error(`Unsupported token type: ${token.type}`); } } -function argToExpression({ +function argToInputDeclaration({ arg: selName, pluralOffset, type -}: SelectArg): Expression { - const arg: VariableRef = { type: 'variable', name: selName }; +}: SelectArg): InputDeclaration { + let functionRef: FunctionRef; if (type === 'select') { - return { - type: 'expression', - arg, - annotation: { type: 'function', name: 'string' } - }; - } + functionRef = { type: 'function', name: 'string' }; + } else { + const options: Options = new Map(); + if (pluralOffset) { + options.set('pluralOffset', { + type: 'literal', + value: String(pluralOffset) + }); + } + if (type === 'selectordinal') { + options.set('type', { type: 'literal', value: 'ordinal' }); + } - const options: Options = new Map(); - if (pluralOffset) { - options.set('pluralOffset', { - type: 'literal', - value: String(pluralOffset) - }); + functionRef = { type: 'function', name: 'mf1:number' }; + if (options.size) functionRef.options = options; } - if (type === 'selectordinal') { - options.set('type', { type: 'literal', value: 'ordinal' }); - } - - const annotation: FunctionAnnotation = { type: 'function', name: 'number' }; - if (options.size) annotation.options = options; - - return { type: 'expression', arg, annotation }; + return { + type: 'input', + name: selName, + value: { + type: 'expression', + arg: { type: 'variable', name: selName }, + functionRef + } + }; } /** @@ -157,7 +146,7 @@ export function mf1ToMessageData(ast: AST.Token[]): Message { return { type: 'message', declarations: [], - pattern: ast.map(token => tokenToPart(token, null, null)) + pattern: ast.map(token => tokenToPart(token, null)) }; } @@ -221,7 +210,7 @@ export function mf1ToMessageData(ast: AST.Token[]): Message { }) ) { const i = vp.length - 1; - const part = tokenToPart(token, pluralArg, pluralOffset); + const part = tokenToPart(token, pluralArg); if (typeof vp[i] === 'string' && typeof part === 'string') { vp[i] += part; } else { @@ -236,8 +225,8 @@ export function mf1ToMessageData(ast: AST.Token[]): Message { return { type: 'select', - declarations: [], - selectors: args.map(argToExpression), + declarations: args.map(argToInputDeclaration), + selectors: args.map(arg => ({ type: 'variable', name: arg.arg })), variants }; } diff --git a/packages/mf2-messageformat/src/cst/declarations.ts b/packages/mf2-messageformat/src/cst/declarations.ts index c5e9e6ab..5df0a1cd 100644 --- a/packages/mf2-messageformat/src/cst/declarations.ts +++ b/packages/mf2-messageformat/src/cst/declarations.ts @@ -1,5 +1,4 @@ -import { parseNameValue } from './names.js'; -import { parseExpression, parseReservedBody } from './expression.js'; +import { parseExpression } from './expression.js'; import type { ParseContext } from './parse-cst.js'; import type * as CST from './types.js'; import { whitespaces } from './util.js'; @@ -16,24 +15,22 @@ export function parseDeclarations( let pos = start; const declarations: CST.Declaration[] = []; loop: while (source[pos] === '.') { - const keyword = parseNameValue(source, pos + 1); + const keyword = source.substr(pos, 6); let decl; switch (keyword) { - case '': - case 'match': + case '.match': break loop; - case 'input': + case '.input': decl = parseInputDeclaration(ctx, pos); break; - case 'local': + case '.local': decl = parseLocalDeclaration(ctx, pos); break; default: - decl = parseReservedStatement(ctx, pos, '.' + keyword); + decl = parseDeclarationJunk(ctx, pos); } declarations.push(decl); - pos = decl.end; - pos += whitespaces(source, pos); + pos = whitespaces(source, decl.end).end; } return { declarations, end: pos }; } @@ -45,7 +42,7 @@ function parseInputDeclaration( // let pos = start + 6; // '.input' const keyword: CST.Syntax<'.input'> = { start, end: pos, value: '.input' }; - pos += whitespaces(ctx.source, pos); + pos = whitespaces(ctx.source, pos).end; const value = parseDeclarationValue(ctx, pos); if (value.type === 'expression') { @@ -66,9 +63,9 @@ function parseLocalDeclaration( let pos = start + 6; // '.local' const keyword: CST.Syntax<'.local'> = { start, end: pos, value: '.local' }; const ws = whitespaces(source, pos); - pos += ws; + pos = ws.end; - if (ws === 0) ctx.onError('missing-syntax', pos, ' '); + if (!ws.hasWS) ctx.onError('missing-syntax', pos, ' '); let target: CST.VariableRef | CST.Junk; if (source[pos] === '$') { @@ -87,7 +84,7 @@ function parseLocalDeclaration( ctx.onError('missing-syntax', junkStart, '$'); } - pos += whitespaces(source, pos); + pos = whitespaces(source, pos).end; let equals: CST.Syntax<'='> | undefined; if (source[pos] === '=') { equals = { start: pos, end: pos + 1, value: '=' }; @@ -96,7 +93,7 @@ function parseLocalDeclaration( ctx.onError('missing-syntax', pos, '='); } - pos += whitespaces(source, pos); + pos = whitespaces(source, pos).end; const value = parseDeclarationValue(ctx, pos); return { @@ -110,47 +107,25 @@ function parseLocalDeclaration( }; } -function parseReservedStatement( - ctx: ParseContext, - start: number, - keyword: string -): CST.ReservedStatement { - let pos = start + keyword.length; - pos += whitespaces(ctx.source, pos); - - const body = parseReservedBody(ctx, pos); - let end = body.end; - pos = end + whitespaces(ctx.source, end); - - const values: CST.Expression[] = []; - while (ctx.source[pos] === '{') { - if (ctx.source.startsWith('{{', pos)) break; - const value = parseExpression(ctx, pos); - values.push(value); - end = value.end; - pos = end + whitespaces(ctx.source, end); - } - if (values.length === 0) ctx.onError('missing-syntax', end, '{'); - - return { - type: 'reserved-statement', - start, - end, - keyword: { start, end: keyword.length, value: keyword }, - body, - values - }; -} - function parseDeclarationValue( ctx: ParseContext, start: number ): CST.Expression | CST.Junk { - const { source } = ctx; - if (source[start] === '{') return parseExpression(ctx, start); + return ctx.source[start] === '{' + ? parseExpression(ctx, start) + : parseDeclarationJunk(ctx, start); +} - const junkEndOffset = source.substring(start).search(/\.[a-z]|{{/); - const end = junkEndOffset === -1 ? source.length : start + junkEndOffset; +function parseDeclarationJunk(ctx: ParseContext, start: number): CST.Junk { + const { source } = ctx; + const junkEndOffset = source.substring(start + 1).search(/\.[a-z]|{{/); + let end: number; + if (junkEndOffset === -1) { + end = source.length; + } else { + end = start + 1 + junkEndOffset; + while (/\s/.test(source[end - 1])) end -= 1; + } ctx.onError('missing-syntax', start, '{'); return { type: 'junk', start, end, source: source.substring(start, end) }; } diff --git a/packages/mf2-messageformat/src/cst/expression.ts b/packages/mf2-messageformat/src/cst/expression.ts index 2182ae27..4415a085 100644 --- a/packages/mf2-messageformat/src/cst/expression.ts +++ b/packages/mf2-messageformat/src/cst/expression.ts @@ -2,8 +2,8 @@ import { MessageSyntaxError } from '../errors.js'; import { parseNameValue } from './names.js'; import type { ParseContext } from './parse-cst.js'; import type * as CST from './types.js'; -import { whitespaceChars, whitespaces } from './util.js'; -import { parseLiteral, parseQuotedLiteral, parseVariable } from './values.js'; +import { whitespaces } from './util.js'; +import { parseLiteral, parseVariable } from './values.js'; export function parseExpression( ctx: ParseContext, @@ -11,7 +11,7 @@ export function parseExpression( ): CST.Expression { const { source } = ctx; let pos = start + 1; // '{' - pos += whitespaces(source, pos); + pos = whitespaces(source, pos).end; const arg = source[pos] === '$' @@ -20,23 +20,19 @@ export function parseExpression( if (arg) { pos = arg.end; const ws = whitespaces(source, pos); - if (ws === 0 && source[pos] !== '}') { + if (!ws.hasWS && source[pos] !== '}') { ctx.onError('missing-syntax', pos, ' '); } - pos += ws; + pos = ws.end; } - let annotation: - | CST.FunctionRef - | CST.ReservedAnnotation - | CST.Junk - | undefined; + let functionRef: CST.FunctionRef | CST.Junk | undefined; let markup: CST.Markup | undefined; let junkError: MessageSyntaxError | undefined; switch (source[pos]) { case ':': - annotation = parseFunctionRefOrMarkup(ctx, pos, 'function'); - pos = annotation.end; + functionRef = parseFunctionRefOrMarkup(ctx, pos, 'function'); + pos = functionRef.end; break; case '#': case '/': @@ -44,19 +40,6 @@ export function parseExpression( markup = parseFunctionRefOrMarkup(ctx, pos, 'markup'); pos = markup.end; break; - case '!': - case '%': - case '^': - case '&': - case '*': - case '+': - case '<': - case '>': - case '?': - case '~': - annotation = parseReservedAnnotation(ctx, pos); - pos = annotation.end; - break; case '@': case '}': if (!arg) ctx.onError('empty-token', start, pos); @@ -64,25 +47,25 @@ export function parseExpression( default: if (!arg) { const end = pos + 1; - annotation = { type: 'junk', start: pos, end, source: source[pos] }; + functionRef = { type: 'junk', start: pos, end, source: source[pos] }; junkError = new MessageSyntaxError('parse-error', start, end); ctx.errors.push(junkError); } } const attributes: CST.Attribute[] = []; - let reqWS = Boolean(annotation || markup); + let reqWS = Boolean(functionRef || markup); let ws = whitespaces(source, pos); - while (source[pos + ws] === '@') { - if (reqWS && ws === 0) ctx.onError('missing-syntax', pos, ' '); - pos += ws; + while (source[ws.end] === '@') { + if (reqWS && !ws.hasWS) ctx.onError('missing-syntax', pos, ' '); + pos = ws.end; const attr = parseAttribute(ctx, pos); attributes.push(attr); pos = attr.end; reqWS = true; ws = whitespaces(source, pos); } - pos += ws; + pos = ws.end; const open: CST.Syntax<'{'> = { start, end: start + 1, value: '{' }; let close: CST.Syntax<'}'> | undefined; @@ -92,9 +75,9 @@ export function parseExpression( if (source[pos] !== '}') { const errStart = pos; while (pos < source.length && source[pos] !== '}') pos += 1; - if (annotation?.type === 'junk') { - annotation.end = pos; - annotation.source = source.substring(annotation.start, pos); + if (functionRef?.type === 'junk') { + functionRef.end = pos; + functionRef.source = source.substring(functionRef.start, pos); if (junkError) junkError.end = pos; } else { ctx.onError('extra-content', errStart, pos); @@ -109,7 +92,15 @@ export function parseExpression( const end = pos; return markup ? { type: 'expression', start, end, braces, markup, attributes } - : { type: 'expression', start, end, braces, arg, annotation, attributes }; + : { + type: 'expression', + start, + end, + braces, + arg, + functionRef, + attributes + }; } function parseFunctionRefOrMarkup( @@ -134,17 +125,17 @@ function parseFunctionRefOrMarkup( let close: CST.Syntax<'/'> | undefined; while (pos < source.length) { let ws = whitespaces(source, pos); - const next = source[pos + ws]; + const next = source[ws.end]; if (next === '@' || next === '}') break; if (next === '/' && source[start] === '#') { - pos += ws + 1; + pos = ws.end + 1; close = { start: pos - 1, end: pos, value: '/' }; ws = whitespaces(source, pos); - if (ws > 0) ctx.onError('extra-content', pos, pos + ws); + if (ws.hasWS) ctx.onError('extra-content', pos, ws.end); break; } - if (ws === 0) ctx.onError('missing-syntax', pos, ' '); - pos += ws; + if (!ws.hasWS) ctx.onError('missing-syntax', pos, ' '); + pos = ws.end; const opt = parseOption(ctx, pos); if (opt.end === pos) break; // error options.push(opt); @@ -161,8 +152,7 @@ function parseFunctionRefOrMarkup( function parseOption(ctx: ParseContext, start: number): CST.Option { const id = parseIdentifier(ctx, start); - let pos = id.end; - pos += whitespaces(ctx.source, pos); + let pos = whitespaces(ctx.source, id.end).end; let equals: CST.Syntax<'='> | undefined; if (ctx.source[pos] === '=') { equals = { start: pos, end: pos + 1, value: '=' }; @@ -170,7 +160,7 @@ function parseOption(ctx: ParseContext, start: number): CST.Option { } else { ctx.onError('missing-syntax', pos, '='); } - pos += whitespaces(ctx.source, pos); + pos = whitespaces(ctx.source, pos).end; const value = ctx.source[pos] === '$' ? parseVariable(ctx, pos) @@ -183,94 +173,28 @@ function parseIdentifier( start: number ): { parts: CST.Identifier; end: number } { const { source } = ctx; - const str0 = parseNameValue(source, start); - if (!str0) { + const name0 = parseNameValue(source, start); + if (!name0) { ctx.onError('empty-token', start, start + 1); return { parts: [{ start, end: start, value: '' }], end: start }; } - let pos = start + str0.length; - const id0 = { start, end: pos, value: str0 }; + let pos = name0.end; + const id0 = { start, end: pos, value: name0.value }; if (source[pos] !== ':') return { parts: [id0], end: pos }; const sep = { start: pos, end: pos + 1, value: ':' as const }; pos += 1; - const str1 = parseNameValue(source, pos); - if (str1) { - const end = pos + str1.length; - const id1 = { start: pos, end, value: str1 }; - return { parts: [id0, sep, id1], end }; + const name1 = parseNameValue(source, pos); + if (name1) { + const id1 = { start: pos, end: name1.end, value: name1.value }; + return { parts: [id0, sep, id1], end: name1.end }; } else { ctx.onError('empty-token', pos, pos + 1); return { parts: [id0, sep], end: pos }; } } -function parseReservedAnnotation( - ctx: ParseContext, - start: number -): CST.ReservedAnnotation { - const open = { - start, - end: start + 1, - value: ctx.source[start] - } as CST.ReservedAnnotation['open']; - const source = parseReservedBody(ctx, start + 1); // skip sigil - return { - type: 'reserved-annotation', - start, - end: source.end, - open, - source - }; -} - -export function parseReservedBody( - ctx: ParseContext, - start: number -): CST.Syntax { - let pos = start; - loop: while (pos < ctx.source.length) { - const ch = ctx.source[pos]; - switch (ch) { - case '\\': { - switch (ctx.source[pos + 1]) { - case '\\': - case '{': - case '|': - case '}': - break; - default: - ctx.onError('bad-escape', pos, pos + 2); - } - pos += 2; - break; - } - case '|': - pos = parseQuotedLiteral(ctx, pos).end; - break; - case '@': - case '{': - case '}': - break loop; - default: { - const cc = ch.charCodeAt(0); - if (cc >= 0xd800 && cc < 0xe000) { - // surrogates are invalid here - ctx.onError('parse-error', pos, pos + 1); - } - pos += 1; - } - } - } - let prev = ctx.source[pos - 1]; - while (pos > start && whitespaceChars.includes(prev)) { - pos -= 1; - prev = ctx.source[pos - 1]; - } - return { start, end: pos, value: ctx.source.substring(start, pos) }; -} - function parseAttribute(ctx: ParseContext, start: number): CST.Attribute { const { source } = ctx; const id = parseIdentifier(ctx, start + 1); @@ -278,10 +202,10 @@ function parseAttribute(ctx: ParseContext, start: number): CST.Attribute { const ws = whitespaces(source, pos); let equals: CST.Syntax<'='> | undefined; let value: CST.Literal | undefined; - if (source[pos + ws] === '=') { - pos += ws + 1; + if (source[ws.end] === '=') { + pos = ws.end + 1; equals = { start: pos - 1, end: pos, value: '=' }; - pos += whitespaces(source, pos); + pos = whitespaces(source, pos).end; value = parseLiteral(ctx, pos, true); pos = value.end; } diff --git a/packages/mf2-messageformat/src/cst/names.ts b/packages/mf2-messageformat/src/cst/names.ts index 80e9b6e5..a5c668cc 100644 --- a/packages/mf2-messageformat/src/cst/names.ts +++ b/packages/mf2-messageformat/src/cst/names.ts @@ -1,3 +1,13 @@ +const bidiChars = new Set([ + 0x061c, // ALM + 0x200e, // LRM + 0x200f, // RLM + 0x2066, // LRI + 0x2067, // RLI + 0x2068, // FSI + 0x2069 // PDI +]); + const isNameStartCode = (cc: number) => (cc >= 0x41 && cc <= 0x5a) || // A-Z cc === 0x5f || // _ @@ -28,11 +38,24 @@ const isNameCharCode = (cc: number) => // This is sticky so that parsing doesn't need to substring the source const numberLiteral = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][-+]?\d+)?/y; -export function parseNameValue(src: string, start: number): string { - if (!isNameStartCode(src.charCodeAt(start))) return ''; - let pos = start + 1; - while (isNameCharCode(src.charCodeAt(pos))) pos += 1; - return src.substring(start, pos); +export function parseNameValue( + src: string, + start: number +): { value: string; end: number } | null { + let pos = start; + let nameStart = start; + let cc = src.charCodeAt(start); + if (bidiChars.has(cc)) { + pos += 1; + nameStart += 1; + cc = src.charCodeAt(pos); + } + if (!isNameStartCode(cc)) return null; + cc = src.charCodeAt(++pos); + while (isNameCharCode(cc)) cc = src.charCodeAt(++pos); + const name = src.substring(nameStart, pos); + if (bidiChars.has(cc)) pos += 1; + return { value: name, end: pos }; } export function isValidUnquotedLiteral(str: string): boolean { diff --git a/packages/mf2-messageformat/src/cst/parse-cst.ts b/packages/mf2-messageformat/src/cst/parse-cst.ts index 506baf72..89375553 100644 --- a/packages/mf2-messageformat/src/cst/parse-cst.ts +++ b/packages/mf2-messageformat/src/cst/parse-cst.ts @@ -3,7 +3,7 @@ import { parseDeclarations } from './declarations.js'; import { parseExpression } from './expression.js'; import type * as CST from './types.js'; import { whitespaces } from './util.js'; -import { parseLiteral, parseText } from './values.js'; +import { parseLiteral, parseText, parseVariable } from './values.js'; export class ParseContext { readonly errors: MessageSyntaxError[] = []; @@ -52,7 +52,7 @@ export function parseCST( ): CST.Message { const ctx = new ParseContext(source, opt); - const pos = whitespaces(source, 0); + const pos = whitespaces(source, 0).end; if (source.startsWith('.', pos)) { const { declarations, end } = parseDeclarations(ctx, pos); return source.startsWith('.match', end) @@ -72,8 +72,7 @@ function parsePatternMessage( complex: boolean ): CST.SimpleMessage | CST.ComplexMessage { const pattern = parsePattern(ctx, start, complex); - let pos = pattern.end; - pos += whitespaces(ctx.source, pos); + const pos = whitespaces(ctx.source, pattern.end).end; if (pos < ctx.source.length) { ctx.onError('extra-content', pos, ctx.source.length); @@ -91,30 +90,31 @@ function parseSelectMessage( ): CST.SelectMessage { let pos = start + 6; // '.match' const match: CST.Syntax<'.match'> = { start, end: pos, value: '.match' }; - pos += whitespaces(ctx.source, pos); - - const selectors: CST.Expression[] = []; - while (ctx.source[pos] === '{') { - const sel = parseExpression(ctx, pos); - const body = sel.markup ?? sel.annotation; - if (body && body.type !== 'function') { - ctx.onError('parse-error', body.start, body.end); - } + let ws = whitespaces(ctx.source, pos); + if (!ws.hasWS) ctx.onError('missing-syntax', pos, "' '"); + pos = ws.end; + + const selectors: CST.VariableRef[] = []; + while (ctx.source[pos] === '$') { + const sel = parseVariable(ctx, pos); selectors.push(sel); pos = sel.end; - pos += whitespaces(ctx.source, pos); - } - if (selectors.length === 0) { - ctx.onError('empty-token', pos, pos + 1); + ws = whitespaces(ctx.source, pos); + if (!ws.hasWS) ctx.onError('missing-syntax', pos, "' '"); + pos = ws.end; } + if (selectors.length === 0) ctx.onError('empty-token', pos, pos + 1); const variants: CST.Variant[] = []; - pos += whitespaces(ctx.source, pos); while (pos < ctx.source.length) { const variant = parseVariant(ctx, pos); - variants.push(variant); - pos = variant.end; - pos += whitespaces(ctx.source, pos); + if (variant.end > pos) { + variants.push(variant); + pos = variant.end; + } else { + pos += 1; + } + pos = whitespaces(ctx.source, pos).end; } if (pos < ctx.source.length) { @@ -136,11 +136,11 @@ function parseVariant(ctx: ParseContext, start: number): CST.Variant { const keys: Array = []; while (pos < ctx.source.length) { const ws = whitespaces(ctx.source, pos); - pos += ws; + pos = ws.end; const ch = ctx.source[pos]; if (ch === '{') break; - if (pos > start && ws === 0) ctx.onError('missing-syntax', pos, "' '"); + if (pos > start && !ws.hasWS) ctx.onError('missing-syntax', pos, "' '"); const key = ch === '*' diff --git a/packages/mf2-messageformat/src/cst/resource-option.test.ts b/packages/mf2-messageformat/src/cst/resource-option.test.ts index 68c42ff6..3985a33d 100644 --- a/packages/mf2-messageformat/src/cst/resource-option.test.ts +++ b/packages/mf2-messageformat/src/cst/resource-option.test.ts @@ -51,7 +51,7 @@ describe('messages in resources', () => { end: src.length - 1, value: '\t \n\r\t\x01\x02\x03' }, - annotation: undefined, + functionRef: undefined, attributes: [] } ] diff --git a/packages/mf2-messageformat/src/cst/stringify-cst.ts b/packages/mf2-messageformat/src/cst/stringify-cst.ts index 8b78c41c..6dbc40f4 100644 --- a/packages/mf2-messageformat/src/cst/stringify-cst.ts +++ b/packages/mf2-messageformat/src/cst/stringify-cst.ts @@ -11,34 +11,31 @@ export function stringifyCST(cst: CST.Message): string { if (cst.declarations) { for (const decl of cst.declarations) { - const kw = decl.keyword.value; switch (decl.type) { case 'input': { + const kw = decl.keyword.value; const val = stringifyExpression(decl.value); str += `${kw} ${val}\n`; break; } case 'local': { + const kw = decl.keyword.value; const tgt = stringifyValue(decl.target); const eq = decl.equals?.value ?? '='; const val = stringifyExpression(decl.value); str += `${kw} ${tgt} ${eq} ${val}\n`; break; } - case 'reserved-statement': { - str += kw; - if (decl.body.value) str += ' ' + decl.body.value; - for (const exp of decl.values) str += ' ' + stringifyExpression(exp); - str += '\n'; + case 'junk': + str += decl.source + '\n'; break; - } } } } if (cst.type === 'select') { str += cst.match.value; - for (const sel of cst.selectors) str += ' ' + stringifyExpression(sel); + for (const sel of cst.selectors) str += ' ' + stringifyValue(sel); for (const { keys, value } of cst.variants) { str += '\n'; for (const key of keys) str += stringifyValue(key) + ' '; @@ -73,7 +70,7 @@ function stringifyExpression( exp: CST.Expression | CST.Junk | undefined ): string { if (exp?.type !== 'expression') return exp?.source ?? ''; - const { braces, arg, annotation, markup, attributes } = exp; + const { braces, arg, functionRef, markup, attributes } = exp; let str = braces[0]?.value ?? '{'; if (markup) { str += markup.open.value + stringifyIdentifier(markup.name); @@ -84,18 +81,15 @@ function stringifyExpression( } else { if (arg) { str += stringifyValue(arg); - if (annotation) str += ' '; + if (functionRef) str += ' '; } - switch (annotation?.type) { + switch (functionRef?.type) { case 'function': - str += annotation.open.value + stringifyIdentifier(annotation.name); - for (const opt of annotation.options) str += stringifyOption(opt); - break; - case 'reserved-annotation': - str += annotation.open.value + annotation.source.value; + str += functionRef.open.value + stringifyIdentifier(functionRef.name); + for (const opt of functionRef.options) str += stringifyOption(opt); break; case 'junk': - str += annotation.source; + str += functionRef.source; break; } } diff --git a/packages/mf2-messageformat/src/cst/types.ts b/packages/mf2-messageformat/src/cst/types.ts index e794343a..683544cd 100644 --- a/packages/mf2-messageformat/src/cst/types.ts +++ b/packages/mf2-messageformat/src/cst/types.ts @@ -24,16 +24,13 @@ export interface SelectMessage { type: 'select'; declarations: Declaration[]; match: Syntax<'.match'>; - selectors: Expression[]; + selectors: VariableRef[]; variants: Variant[]; errors: MessageSyntaxError[]; } /** @beta */ -export type Declaration = - | InputDeclaration - | LocalDeclaration - | ReservedStatement; +export type Declaration = InputDeclaration | LocalDeclaration | Junk; /** @beta */ export interface InputDeclaration { @@ -55,16 +52,6 @@ export interface LocalDeclaration { value: Expression | Junk; } -/** @beta */ -export interface ReservedStatement { - type: 'reserved-statement'; - start: number; - end: number; - keyword: Syntax; - body: Syntax; - values: Expression[]; -} - /** @beta */ export interface Variant { start: number; @@ -104,7 +91,7 @@ export interface Expression { end: number; braces: [Syntax<'{'>] | [Syntax<'{'>, Syntax<'}'>]; arg?: Literal | VariableRef; - annotation?: FunctionRef | ReservedAnnotation | Junk; + functionRef?: FunctionRef | Junk; markup?: Markup; attributes: Attribute[]; } @@ -148,15 +135,6 @@ export interface FunctionRef { options: Option[]; } -/** @beta */ -export interface ReservedAnnotation { - type: 'reserved-annotation'; - open: Syntax<'!' | '%' | '^' | '&' | '*' | '+' | '<' | '>' | '?' | '~'>; - source: Syntax; - start: number; - end: number; -} - /** @beta */ export interface Markup { type: 'markup'; diff --git a/packages/mf2-messageformat/src/cst/util.ts b/packages/mf2-messageformat/src/cst/util.ts index 0a7e0854..a04a8395 100644 --- a/packages/mf2-messageformat/src/cst/util.ts +++ b/packages/mf2-messageformat/src/cst/util.ts @@ -1,11 +1,18 @@ -export const whitespaceChars = ['\t', '\n', '\r', ' ', '\u3000']; +const bidiChars = new Set('\u061C\u200E\u200F\u2066\u2067\u2068\u2069'); +const whitespaceChars = new Set('\t\n\r \u3000'); -export function whitespaces(src: string, start: number): number { - let length = 0; - let ch = src[start]; - while (whitespaceChars.includes(ch)) { - length += 1; - ch = src[start + length]; +export function whitespaces( + src: string, + start: number +): { hasWS: boolean; end: number } { + let hasWS = false; + let pos = start; + let ch = src[pos]; + while (bidiChars.has(ch)) ch = src[++pos]; + while (whitespaceChars.has(ch)) { + hasWS = true; + ch = src[++pos]; } - return length; + while (bidiChars.has(ch) || whitespaceChars.has(ch)) ch = src[++pos]; + return { hasWS, end: pos }; } diff --git a/packages/mf2-messageformat/src/cst/values.ts b/packages/mf2-messageformat/src/cst/values.ts index bbbdd42a..14aa25bb 100644 --- a/packages/mf2-messageformat/src/cst/values.ts +++ b/packages/mf2-messageformat/src/cst/values.ts @@ -70,10 +70,7 @@ export function parseLiteral( return { type: 'literal', quoted: false, start, end, value }; } -export function parseQuotedLiteral( - ctx: ParseContext, - start: number -): CST.Literal { +function parseQuotedLiteral(ctx: ParseContext, start: number): CST.Literal { let value = ''; let pos = start + 1; const open = { start, end: pos, value: '|' as const }; @@ -135,9 +132,11 @@ export function parseVariable( const pos = start + 1; const open = { start, end: pos, value: '$' as const }; const name = parseNameValue(ctx.source, pos); - const end = pos + name.length; - if (!name) ctx.onError('empty-token', pos, pos + 1); - return { type: 'variable', start, end, open, name }; + if (!name) { + ctx.onError('empty-token', pos, pos + 1); + return { type: 'variable', start, end: pos, open, name: '' }; + } + return { type: 'variable', start, end: name.end, open, name: name.value }; } function parseEscape( diff --git a/packages/mf2-messageformat/src/data-model/from-cst.ts b/packages/mf2-messageformat/src/data-model/from-cst.ts index e0ca37ff..e774cbaa 100644 --- a/packages/mf2-messageformat/src/data-model/from-cst.ts +++ b/packages/mf2-messageformat/src/data-model/from-cst.ts @@ -24,7 +24,7 @@ export function messageFromCST(msg: CST.Message): Model.Message { return { type: 'select', declarations, - selectors: msg.selectors.map(sel => asExpression(sel, false)), + selectors: msg.selectors.map(sel => asValue(sel)), variants: msg.variants.map(variant => ({ keys: variant.keys.map(key => key.type === '*' ? { type: '*', [cst]: key } : asValue(key) @@ -67,13 +67,7 @@ function asDeclaration(decl: CST.Declaration): Model.Declaration { [cst]: decl }; default: - return { - type: 'unsupported-statement', - keyword: (decl.keyword?.value ?? '').substring(1), - body: decl.body?.value || undefined, - expressions: decl.values?.map(dv => asExpression(dv, true)) ?? [], - [cst]: decl - }; + throw new MessageSyntaxError('parse-error', decl.start, decl.end); } } @@ -110,35 +104,24 @@ function asExpression( } const arg = exp.arg ? asValue(exp.arg) : undefined; - let annotation: - | Model.FunctionAnnotation - | Model.UnsupportedAnnotation - | undefined; + let functionRef: Model.FunctionRef | undefined; - const ca = exp.annotation; + const ca = exp.functionRef; if (ca) { - switch (ca.type) { - case 'function': - annotation = { type: 'function', name: asName(ca.name) }; - if (ca.options.length) annotation.options = asOptions(ca.options); - break; - case 'reserved-annotation': - annotation = { - type: 'unsupported-annotation', - source: ca.open.value + ca.source.value - }; - break; - default: - throw new MessageSyntaxError('parse-error', exp.start, exp.end); + if (ca.type === 'function') { + functionRef = { type: 'function', name: asName(ca.name) }; + if (ca.options.length) functionRef.options = asOptions(ca.options); + } else { + throw new MessageSyntaxError('parse-error', exp.start, exp.end); } } let expression: Model.Expression | undefined = arg ? { type: 'expression', arg } : undefined; - if (annotation) { - annotation[cst] = ca; - if (expression) expression.annotation = annotation; - else expression = { type: 'expression', annotation }; + if (functionRef) { + functionRef[cst] = ca; + if (expression) expression.functionRef = functionRef; + else expression = { type: 'expression', functionRef: functionRef }; } if (expression) { if (exp.attributes.length) { diff --git a/packages/mf2-messageformat/src/data-model/function-annotation.test.ts b/packages/mf2-messageformat/src/data-model/function-ref.test.ts similarity index 100% rename from packages/mf2-messageformat/src/data-model/function-annotation.test.ts rename to packages/mf2-messageformat/src/data-model/function-ref.test.ts diff --git a/packages/mf2-messageformat/src/data-model/parse.test.ts b/packages/mf2-messageformat/src/data-model/parse.test.ts deleted file mode 100644 index 9b44ed3d..00000000 --- a/packages/mf2-messageformat/src/data-model/parse.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { parseMessage } from '../index.js'; - -describe('private annotations', () => { - for (const source of ['{^foo}', '{&bar}']) { - test(source, () => { - const data = parseMessage(source, { - privateAnnotation(src, pos) { - const end = src.indexOf('}', pos); - return { - annotation: { type: 'private' }, - pos: end - }; - } - }); - expect(data).toEqual({ - type: 'message', - declarations: [], - pattern: [{ type: 'expression', annotation: { type: 'private' } }] - }); - }); - } - - test('foo {&bar @baz}', () => { - const data = parseMessage('foo {&bar @baz}', { - privateAnnotation(src, pos) { - const end = src.indexOf(' ', pos); - return { - annotation: { type: 'priv-bar' }, - pos: end - }; - } - }); - expect(data).toEqual({ - type: 'message', - declarations: [], - pattern: [ - 'foo ', - { - type: 'expression', - annotation: { type: 'priv-bar' }, - attributes: new Map([['baz', true]]) - } - ] - }); - }); -}); diff --git a/packages/mf2-messageformat/src/data-model/parse.ts b/packages/mf2-messageformat/src/data-model/parse.ts index e2b29bb1..5e3a20ad 100644 --- a/packages/mf2-messageformat/src/data-model/parse.ts +++ b/packages/mf2-messageformat/src/data-model/parse.ts @@ -2,28 +2,13 @@ import { parseNameValue, parseUnquotedLiteralValue } from '../cst/names.js'; import { MessageSyntaxError } from '../errors.js'; import type * as Model from './types.js'; -const whitespaceChars = ['\t', '\n', '\r', ' ', '\u3000']; - -export type MessageParserOptions = { - /** - * Parse a private annotation starting with `^` or `&`. - * By default, private annotations are parsed as unsupported annotations. - * - * @returns `pos` as the position at the end of the `annotation`, - * not including any trailing whitespace. - */ - privateAnnotation?: ( - source: string, - pos: number - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ) => { pos: number; annotation: any }; -}; +const bidiChars = new Set('\u061C\u200E\u200F\u2066\u2067\u2068\u2069'); +const whitespaceChars = new Set('\t\n\r \u3000'); //// Parser State //// let pos: number; let source: string; -let opt: MessageParserOptions; //// Utilities & Error Wrappers //// @@ -56,17 +41,10 @@ function expect(searchString: string, consume: boolean) { * its corresponding data model representation. * Throws on syntax errors, but does not check for data model errors. */ -export function parseMessage( - source: string, - opt?: MessageParserOptions -): Model.Message; -export function parseMessage( - source_: string, - opt_: MessageParserOptions = {} -): Model.Message { +export function parseMessage(source: string): Model.Message; +export function parseMessage(source_: string): Model.Message { pos = 0; source = source_; - opt = opt_; const decl = declarations(); if (source.startsWith('.match', pos)) return selectMessage(decl); @@ -85,12 +63,12 @@ export function parseMessage( function selectMessage(declarations: Model.Declaration[]): Model.SelectMessage { pos += 6; // '.match' - ws(); + ws(true); - const selectors: Model.Expression[] = []; - while (source[pos] === '{') { - selectors.push(expression(false)); - ws(); + const selectors: Model.VariableRef[] = []; + while (source[pos] === '$') { + selectors.push(variable()); + ws(true); } if (selectors.length === 0) throw SyntaxError('empty-token', pos); @@ -152,20 +130,18 @@ function declarations(): Model.Declaration[] { const declarations: Model.Declaration[] = []; ws(); loop: while (source[pos] === '.') { - const keyword = parseNameValue(source, pos + 1); + const keyword = source.substr(pos, 6); switch (keyword) { - case 'input': + case '.input': declarations.push(inputDeclaration()); break; - case 'local': + case '.local': declarations.push(localDeclaration()); break; - case 'match': + case '.match': break loop; - case '': - throw SyntaxError('parse-error', pos); default: - declarations.push(unsupportedStatement(keyword)); + throw SyntaxError('parse-error', pos); } ws(); } @@ -198,20 +174,6 @@ function localDeclaration(): Model.LocalDeclaration { return { type: 'local', name: name_, value }; } -function unsupportedStatement(keyword: string): Model.UnsupportedStatement { - pos += 1 + keyword.length; // '.' + keyword - ws('{'); - const body = reservedBody() || undefined; - const expressions: (Model.Expression | Model.Markup)[] = []; - while (source[pos] === '{') { - if (source.startsWith('{{', pos)) break; - expressions.push(expression(false)); - ws(); - } - if (expressions.length === 0) throw SyntaxError('empty-token', pos); - return { type: 'unsupported-statement', keyword, body, expressions }; -} - function expression(allowMarkup: false): Model.Expression; function expression(allowMarkup: boolean): Model.Expression | Model.Markup; function expression(allowMarkup: boolean): Model.Expression | Model.Markup { @@ -223,10 +185,7 @@ function expression(allowMarkup: boolean): Model.Expression | Model.Markup { if (arg) ws('}'); const sigil = source[pos]; - let annotation: - | Model.FunctionAnnotation - | Model.UnsupportedAnnotation - | undefined; + let functionRef: Model.FunctionRef | undefined; let markup: Model.Markup | undefined; switch (sigil) { case '@': @@ -234,9 +193,9 @@ function expression(allowMarkup: boolean): Model.Expression | Model.Markup { break; case ':': { pos += 1; // ':' - annotation = { type: 'function', name: identifier() }; + functionRef = { type: 'function', name: identifier() }; const options_ = options(); - if (options_) annotation.options = options_; + if (options_) functionRef.options = options_; break; } case '#': @@ -249,20 +208,6 @@ function expression(allowMarkup: boolean): Model.Expression | Model.Markup { if (options_) markup.options = options_; break; } - case '^': - case '&': - annotation = privateAnnotation(sigil); - break; - case '!': - case '%': - case '*': - case '+': - case '<': - case '>': - case '?': - case '~': - annotation = unsupportedAnnotation(sigil); - break; default: throw SyntaxError('parse-error', pos); } @@ -274,10 +219,10 @@ function expression(allowMarkup: boolean): Model.Expression | Model.Markup { } expect('}', true); - if (annotation) { + if (functionRef) { const exp: Model.Expression = arg - ? { type: 'expression', arg, annotation } - : { type: 'expression', annotation }; + ? { type: 'expression', arg, functionRef: functionRef } + : { type: 'expression', functionRef: functionRef }; if (attributes_) exp.attributes = attributes_; return exp; } @@ -338,63 +283,6 @@ function attributes() { return isEmpty ? null : attributes; } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function privateAnnotation(sigil: '^' | '&'): any { - if (opt.privateAnnotation) { - const res = opt.privateAnnotation(source, pos); - pos = res.pos; - ws('}'); - return res.annotation; - } - return unsupportedAnnotation(sigil); -} - -function unsupportedAnnotation(sigil: string): Model.UnsupportedAnnotation { - pos += 1; // sigil - return { type: 'unsupported-annotation', source: sigil + reservedBody() }; -} - -function reservedBody(): string { - const start = pos; - loop: while (pos < source.length) { - const next = source[pos]; - switch (next) { - case '\\': { - switch (source[pos + 1]) { - case '\\': - case '{': - case '|': - case '}': - break; - default: - throw SyntaxError('bad-escape', pos, pos + 2); - } - pos += 2; - break; - } - case '|': - quotedLiteral(); - break; - case '@': - pos -= 1; - ws(true); - break loop; - case '{': - case '}': - break loop; - default: { - const cc = next.charCodeAt(0); - if (cc >= 0xd800 && cc < 0xe000) { - // surrogates are invalid here - throw SyntaxError('parse-error', pos); - } - pos += 1; - } - } - } - return source.substring(start, pos).trimEnd(); -} - function text(): string { let value = ''; let i = pos; @@ -425,12 +313,12 @@ function value( function value( required: boolean ): Model.Literal | Model.VariableRef | undefined { - if (source[pos] === '$') { - pos += 1; // '$' - return { type: 'variable', name: name() }; - } else { - return literal(required); - } + return source[pos] === '$' ? variable() : literal(required); +} + +function variable(): Model.VariableRef { + pos += 1; // '$' + return { type: 'variable', name: name() }; } function literal(required: true): Model.Literal; @@ -480,22 +368,25 @@ function identifier(): string { function name(): string { const name = parseNameValue(source, pos); if (!name) throw SyntaxError('empty-token', pos); - pos += name.length; - return name; + pos = name.end; + return name.value; } function ws(required?: boolean): void; function ws(requiredIfNotFollowedBy: string): void; -function ws(req: string | boolean): void; +function ws(required: string | boolean): void; function ws(req: string | boolean = false): void { - let length = 0; let next = source[pos]; - while (whitespaceChars.includes(next)) { - length += 1; - next = source[pos + length]; + let hasWS = false; + if (req) { + while (bidiChars.has(next)) next = source[++pos]; + while (whitespaceChars.has(next)) { + next = source[++pos]; + hasWS = true; + } } - pos += length; - if (req && !length && (req === true || !req.includes(source[pos]))) { + while (bidiChars.has(next) || whitespaceChars.has(next)) next = source[++pos]; + if (req && !hasWS && (req === true || !req.includes(source[pos]))) { throw MissingSyntax(pos, "' '"); } } diff --git a/packages/mf2-messageformat/src/data-model/resolve-expression.ts b/packages/mf2-messageformat/src/data-model/resolve-expression.ts index d94f2a7f..fa5c530f 100644 --- a/packages/mf2-messageformat/src/data-model/resolve-expression.ts +++ b/packages/mf2-messageformat/src/data-model/resolve-expression.ts @@ -1,19 +1,16 @@ import type { Context } from '../format-context.js'; import type { MessageValue } from '../functions/index.js'; -import { resolveFunctionAnnotation } from './resolve-function-annotation.js'; +import { resolveFunctionRef } from './resolve-function-ref.js'; import { resolveLiteral } from './resolve-literal.js'; -import { resolveUnsupportedAnnotation } from './resolve-unsupported-annotation.js'; import { resolveVariableRef } from './resolve-variable.js'; import type { Expression } from './types.js'; export function resolveExpression( ctx: Context, - { arg, annotation }: Expression + { arg, functionRef }: Expression ): MessageValue { - if (annotation) { - return annotation.type === 'function' - ? resolveFunctionAnnotation(ctx, arg, annotation) - : resolveUnsupportedAnnotation(ctx, arg, annotation); + if (functionRef) { + return resolveFunctionRef(ctx, arg, functionRef); } switch (arg?.type) { case 'literal': diff --git a/packages/mf2-messageformat/src/data-model/resolve-function-annotation.ts b/packages/mf2-messageformat/src/data-model/resolve-function-ref.ts similarity index 86% rename from packages/mf2-messageformat/src/data-model/resolve-function-annotation.ts rename to packages/mf2-messageformat/src/data-model/resolve-function-ref.ts index c300defe..438517cb 100644 --- a/packages/mf2-messageformat/src/data-model/resolve-function-annotation.ts +++ b/packages/mf2-messageformat/src/data-model/resolve-function-ref.ts @@ -3,17 +3,12 @@ import type { Context } from '../format-context.js'; import { fallback } from '../functions/fallback.js'; import { MessageFunctionContext } from './function-context.js'; import { getValueSource, resolveValue } from './resolve-value.js'; -import type { - FunctionAnnotation, - Literal, - Options, - VariableRef -} from './types.js'; +import type { FunctionRef, Literal, Options, VariableRef } from './types.js'; -export function resolveFunctionAnnotation( +export function resolveFunctionRef( ctx: Context, operand: Literal | VariableRef | undefined, - { name, options }: FunctionAnnotation + { name, options }: FunctionRef ) { let source: string | undefined; try { @@ -34,7 +29,8 @@ export function resolveFunctionAnnotation( const opt = resolveOptions(ctx, options); const res = rf(msgCtx, opt, ...fnInput); if ( - !(res instanceof Object) || + res === null || + (typeof res !== 'object' && typeof res !== 'function') || typeof res.type !== 'string' || typeof res.source !== 'string' ) { diff --git a/packages/mf2-messageformat/src/data-model/resolve-unsupported-annotation.ts b/packages/mf2-messageformat/src/data-model/resolve-unsupported-annotation.ts deleted file mode 100644 index 3ad7f681..00000000 --- a/packages/mf2-messageformat/src/data-model/resolve-unsupported-annotation.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { MessageResolutionError } from '../errors.js'; -import type { Context } from '../format-context.js'; -import { fallback } from '../functions/fallback.js'; -import { getValueSource } from './resolve-value.js'; -import type { Literal, UnsupportedAnnotation, VariableRef } from './types.js'; - -export function resolveUnsupportedAnnotation( - ctx: Context, - operand: Literal | VariableRef | undefined, - { source = '�' }: UnsupportedAnnotation -) { - const sigil = source[0]; - const msg = `Reserved ${sigil} annotation is not supported`; - ctx.onError(new MessageResolutionError('unsupported-expression', msg, sigil)); - return fallback(getValueSource(operand) ?? sigil); -} diff --git a/packages/mf2-messageformat/src/data-model/resolve-variable.ts b/packages/mf2-messageformat/src/data-model/resolve-variable.ts index 5fc2f396..ac5c044b 100644 --- a/packages/mf2-messageformat/src/data-model/resolve-variable.ts +++ b/packages/mf2-messageformat/src/data-model/resolve-variable.ts @@ -24,7 +24,7 @@ export class UnresolvedExpression { } const isScope = (scope: unknown): scope is Record => - scope instanceof Object; + scope !== null && (typeof scope === 'object' || typeof scope === 'function'); /** * Looks for the longest matching `.` delimited starting substring of name. diff --git a/packages/mf2-messageformat/src/data-model/stringify.ts b/packages/mf2-messageformat/src/data-model/stringify.ts index c8ed4ebf..e44ca979 100644 --- a/packages/mf2-messageformat/src/data-model/stringify.ts +++ b/packages/mf2-messageformat/src/data-model/stringify.ts @@ -8,7 +8,7 @@ import { import type { Declaration, Expression, - FunctionAnnotation, + FunctionRef, Literal, Markup, Message, @@ -28,7 +28,7 @@ export function stringifyMessage(msg: Message) { res += stringifyPattern(msg.pattern, !!res); } else if (isSelectMessage(msg)) { res += '.match'; - for (const sel of msg.selectors) res += ' ' + stringifyExpression(sel); + for (const sel of msg.selectors) res += ' ' + stringifyVariableRef(sel); for (const { keys, value } of msg.variants) { res += '\n'; for (const key of keys) { @@ -46,24 +46,12 @@ function stringifyDeclaration(decl: Declaration) { return `.input ${stringifyExpression(decl.value)}\n`; case 'local': return `.local $${decl.name} = ${stringifyExpression(decl.value)}\n`; - case 'unsupported-statement': { - const parts = [`.${decl.keyword}`]; - if (decl.body) parts.push(decl.body); - for (const exp of decl.expressions) { - parts.push( - exp.type === 'expression' - ? stringifyExpression(exp) - : stringifyMarkup(exp) - ); - } - return parts.join(' ') + '\n'; - } } // @ts-expect-error Guard against non-TS users with bad data throw new Error(`Unsupported ${decl.type} declaration`); } -function stringifyFunctionAnnotation({ name, options }: FunctionAnnotation) { +function stringifyFunctionRef({ name, options }: FunctionRef) { let res = `:${name}`; if (options) { for (const [key, value] of options) { @@ -125,7 +113,7 @@ function stringifyString(str: string, quoted: boolean) { return str.replace(esc, '\\$&'); } -function stringifyExpression({ arg, annotation, attributes }: Expression) { +function stringifyExpression({ arg, attributes, functionRef }: Expression) { let res: string; switch (arg?.type) { case 'literal': @@ -137,12 +125,9 @@ function stringifyExpression({ arg, annotation, attributes }: Expression) { default: res = ''; } - if (annotation) { + if (functionRef) { if (res) res += ' '; - res += - annotation.type === 'function' - ? stringifyFunctionAnnotation(annotation) - : annotation.source ?? '�'; + res += stringifyFunctionRef(functionRef); } if (attributes) { for (const [name, value] of attributes) { diff --git a/packages/mf2-messageformat/src/data-model/type-guards.ts b/packages/mf2-messageformat/src/data-model/type-guards.ts index 8234f1d6..b0c46999 100644 --- a/packages/mf2-messageformat/src/data-model/type-guards.ts +++ b/packages/mf2-messageformat/src/data-model/type-guards.ts @@ -3,13 +3,12 @@ import type { CatchallKey, Expression, - FunctionAnnotation, + FunctionRef, Literal, Markup, Message, PatternMessage, SelectMessage, - UnsupportedAnnotation, VariableRef } from './types.js'; @@ -22,7 +21,7 @@ export const isExpression = (part: any): part is Expression => !!part && typeof part === 'object' && part.type === 'expression'; /** @beta */ -export const isFunctionAnnotation = (part: any): part is FunctionAnnotation => +export const isFunctionRef = (part: any): part is FunctionRef => !!part && typeof part === 'object' && part.type === 'function'; /** @beta */ @@ -47,12 +46,6 @@ export const isPatternMessage = (msg: Message): msg is PatternMessage => export const isSelectMessage = (msg: Message): msg is SelectMessage => msg.type === 'select'; -/** @beta */ -export const isUnsupportedAnnotation = ( - part: any -): part is UnsupportedAnnotation => - !!part && typeof part === 'object' && part.type === 'unsupported-annotation'; - /** @beta */ export const isVariableRef = (part: any): part is VariableRef => !!part && typeof part === 'object' && part.type === 'variable'; diff --git a/packages/mf2-messageformat/src/data-model/types.ts b/packages/mf2-messageformat/src/data-model/types.ts index 466953db..3d6bdd66 100644 --- a/packages/mf2-messageformat/src/data-model/types.ts +++ b/packages/mf2-messageformat/src/data-model/types.ts @@ -13,8 +13,7 @@ export type MessageNode = | Expression | Literal | VariableRef - | FunctionAnnotation - | UnsupportedAnnotation + | FunctionRef | Markup; /** @@ -46,10 +45,7 @@ export interface PatternMessage { * * @beta */ -export type Declaration = - | InputDeclaration - | LocalDeclaration - | UnsupportedStatement; +export type Declaration = InputDeclaration | LocalDeclaration; /** @beta */ export interface InputDeclaration { @@ -67,17 +63,6 @@ export interface LocalDeclaration { [cst]?: CST.Declaration; } -/** @beta */ -export interface UnsupportedStatement { - type: 'unsupported-statement'; - keyword: string; - name?: never; - value?: never; - body?: string; - expressions: (Expression | Markup)[]; - [cst]?: CST.Declaration; -} - /** * SelectMessage generalises the plural, selectordinal and select * argument types of MessageFormat 1. @@ -93,7 +78,7 @@ export interface UnsupportedStatement { export interface SelectMessage { type: 'select'; declarations: Declaration[]; - selectors: Expression[]; + selectors: VariableRef[]; variants: Variant[]; comment?: string; [cst]?: CST.SelectMessage; @@ -129,7 +114,7 @@ export type Pattern = Array; /** * Expressions are used in declarations, as selectors, and as placeholders. - * Each must include at least an `arg` or an `annotation`, or both. + * Each must include at least an `arg` or a `functionRef`, or both. * * @beta */ @@ -143,8 +128,8 @@ export type Expression< attributes?: Attributes; [cst]?: CST.Expression; } & (A extends Literal | VariableRef - ? { arg: A; annotation?: FunctionAnnotation | UnsupportedAnnotation } - : { arg?: never; annotation: FunctionAnnotation | UnsupportedAnnotation }); + ? { arg: A; functionRef?: FunctionRef } + : { arg?: never; functionRef: FunctionRef }); /** * An immediately defined value. @@ -182,7 +167,7 @@ export interface VariableRef { } /** - * To resolve a FunctionAnnotation, an externally defined function is called. + * To resolve a FunctionRef, an externally defined function is called. * * @remarks * The `name` identifies a function that takes in the arguments `args`, the @@ -193,31 +178,13 @@ export interface VariableRef { * * @beta */ -export interface FunctionAnnotation { +export interface FunctionRef { type: 'function'; name: string; options?: Options; [cst]?: CST.FunctionRef; } -/** - * When the parser encounters an expression with reserved syntax, - * it emits an UnsupportedAnnotation to represent it. - * - * @remarks - * As the meaning of this syntax is not supported, - * it will always resolve with a fallback representation and emit an error. - * - * @beta - */ -export interface UnsupportedAnnotation { - type: 'unsupported-annotation'; - source: string; - name?: never; - options?: never; - [cst]?: CST.ReservedAnnotation; -} - /** * Markup placeholders can span ranges of other pattern elements, * or represent other inline elements. @@ -241,7 +208,7 @@ export interface Markup { } /** - * The options of {@link FunctionAnnotation} and {@link Markup}. + * The options of {@link FunctionRef} and {@link Markup}. * * @beta */ diff --git a/packages/mf2-messageformat/src/data-model/unsupported-annotation.test.ts b/packages/mf2-messageformat/src/data-model/unsupported-annotation.test.ts deleted file mode 100644 index 06bfd8ac..00000000 --- a/packages/mf2-messageformat/src/data-model/unsupported-annotation.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { MessageFormat } from '../index.js'; - -function resolve( - source: string, - params: Record, - errors: any[] = [] -) { - const mf = new MessageFormat(undefined, source); - const onError = jest.fn(); - const res = mf.formatToParts(params, onError); - expect(onError).toHaveBeenCalledTimes(errors.length); - for (let i = 0; i < errors.length; ++i) { - const err = onError.mock.calls[i][0]; - expect(err).toMatchObject(errors[i]); - } - return res; -} - -describe('Reserved syntax', () => { - test('empty', () => { - const msg = resolve('{!}', {}, [{ type: 'unsupported-expression' }]); - expect(msg).toMatchObject([{ type: 'fallback', source: '!' }]); - }); - - test('argument', () => { - const msg = resolve('{$foo ~bar}', { foo: 42 }, [ - { type: 'unsupported-expression' } - ]); - expect(msg).toMatchObject([{ type: 'fallback', source: '$foo' }]); - }); - - test('attribute', () => { - const msg = resolve('{%bar @foo}', {}, [ - { type: 'unsupported-expression' } - ]); - expect(msg).toMatchObject([{ type: 'fallback', source: '%' }]); - }); - - test('old markup syntax', () => { - const msg = resolve('{+open}', {}, [{ type: 'unsupported-expression' }]); - expect(msg).toMatchObject([{ type: 'fallback', source: '+' }]); - }); - - test('whitespace', () => { - const msg = resolve('{ + one\ntwo\rthree four }', {}, [ - { type: 'unsupported-expression' } - ]); - expect(msg).toMatchObject([{ type: 'fallback', source: '+' }]); - }); - - test('surrogates', () => { - expect( - () => new MessageFormat(undefined, '{ %invalid \ud900 surrogate }') - ).toThrow(); - }); -}); diff --git a/packages/mf2-messageformat/src/data-model/validate.ts b/packages/mf2-messageformat/src/data-model/validate.ts index a83e899a..340cfe56 100644 --- a/packages/mf2-messageformat/src/data-model/validate.ts +++ b/packages/mf2-messageformat/src/data-model/validate.ts @@ -1,5 +1,5 @@ import { MessageDataModelError } from '../errors.js'; -import type { Expression, Message, MessageNode, Variant } from './types.js'; +import type { Message, MessageNode, VariableRef, Variant } from './types.js'; import { visit } from './visit.js'; /** @@ -15,9 +15,8 @@ import { visit } from './visit.js'; * The message does not include a _variant_ with only catch-all keys. * * - **Missing Selector Annotation**: - * A _selector_ does not have an _annotation_, - * or contains a _variable_ that does not directly or indirectly - * reference a _declaration_ with an _annotation_. + * A _selector_ does not contains a _variable_ that directly or indirectly + * reference a _declaration_ with a _function_. * * - **Duplicate Declaration**: * A _variable_ appears in two _declarations_. @@ -41,7 +40,7 @@ export function validate( } ) { let selectorCount = 0; - let missingFallback: Expression | Variant | null = null; + let missingFallback: VariableRef | Variant | null = null; /** Tracks directly & indirectly annotated variables for `missing-selector-annotation` */ const annotated = new Set(); @@ -61,7 +60,7 @@ export function validate( if (!decl.name) return undefined; if ( - decl.value.annotation || + decl.value.functionRef || (decl.type === 'local' && decl.value.arg?.type === 'variable' && annotated.has(decl.value.arg.name)) @@ -78,30 +77,25 @@ export function validate( }; }, - expression(expression, context) { - const { arg, annotation } = expression; - if (annotation?.type === 'function') functions.add(annotation.name); - if (context === 'selector') { - selectorCount += 1; - missingFallback = expression; - if ( - !annotation && - (arg?.type !== 'variable' || !annotated.has(arg.name)) - ) { - onError('missing-selector-annotation', expression); - } - } + expression({ functionRef }) { + if (functionRef) functions.add(functionRef.name); }, value(value, context, position) { - if (value.type === 'variable') { - variables.add(value.name); - if ( - context === 'declaration' && - (position !== 'arg' || setArgAsDeclared) - ) { - declared.add(value.name); - } + if (value.type !== 'variable') return; + variables.add(value.name); + switch (context) { + case 'declaration': + if (position !== 'arg' || setArgAsDeclared) { + declared.add(value.name); + } + break; + case 'selector': + selectorCount += 1; + missingFallback = value; + if (!annotated.has(value.name)) { + onError('missing-selector-annotation', value); + } } }, diff --git a/packages/mf2-messageformat/src/data-model/visit.ts b/packages/mf2-messageformat/src/data-model/visit.ts index b6d1f65f..df3c94d1 100644 --- a/packages/mf2-messageformat/src/data-model/visit.ts +++ b/packages/mf2-messageformat/src/data-model/visit.ts @@ -3,14 +3,13 @@ import type { CatchallKey, Declaration, Expression, - FunctionAnnotation, + FunctionRef, Literal, Markup, Message, MessageNode, Options, Pattern, - UnsupportedAnnotation, VariableRef, Variant } from './types.js'; @@ -34,19 +33,19 @@ import type { export function visit( msg: Message, visitors: { - annotation?: ( - annotation: FunctionAnnotation | UnsupportedAnnotation, - context: 'declaration' | 'selector' | 'placeholder', - argument: Literal | VariableRef | undefined - ) => (() => void) | void; attributes?: ( attributes: Attributes, - context: 'declaration' | 'selector' | 'placeholder' + context: 'declaration' | 'placeholder' ) => (() => void) | void; declaration?: (declaration: Declaration) => (() => void) | void; expression?: ( expression: Expression, - context: 'declaration' | 'selector' | 'placeholder' + context: 'declaration' | 'placeholder' + ) => (() => void) | void; + functionRef?: ( + functionRef: FunctionRef, + context: 'declaration' | 'placeholder', + argument: Literal | VariableRef | undefined ) => (() => void) | void; key?: ( key: Literal | CatchallKey, @@ -60,7 +59,7 @@ export function visit( node?: (node: MessageNode, ...rest: unknown[]) => void; options?: ( options: Options, - context: 'declaration' | 'selector' | 'placeholder' + context: 'declaration' | 'placeholder' ) => (() => void) | void; pattern?: (pattern: Pattern) => (() => void) | void; value?: ( @@ -73,7 +72,7 @@ export function visit( ) { const { node, pattern } = visitors; const { - annotation = node, + functionRef = node, attributes = null, declaration = node, expression = node, @@ -86,7 +85,7 @@ export function visit( const handleOptions = ( options_: Options | undefined, - context: 'declaration' | 'selector' | 'placeholder' + context: 'declaration' | 'placeholder' ) => { if (options_) { const end = options?.(options_, context); @@ -101,7 +100,7 @@ export function visit( const handleAttributes = ( attributes_: Attributes | undefined, - context: 'declaration' | 'selector' | 'placeholder' + context: 'declaration' | 'placeholder' ) => { if (attributes_) { const end = attributes?.(attributes_, context); @@ -116,7 +115,7 @@ export function visit( const handleElement = ( exp: string | Expression | Markup, - context: 'declaration' | 'selector' | 'placeholder' + context: 'declaration' | 'placeholder' ) => { if (typeof exp === 'object') { let end: (() => void) | void | undefined; @@ -124,16 +123,16 @@ export function visit( case 'expression': { end = expression?.(exp, context); if (exp.arg) value?.(exp.arg, context, 'arg'); - if (exp.annotation) { - const endA = annotation?.(exp.annotation, context, exp.arg); - handleOptions(exp.annotation.options, context); + if (exp.functionRef) { + const endA = functionRef?.(exp.functionRef, context, exp.arg); + handleOptions(exp.functionRef.options, context); endA?.(); } handleAttributes(exp.attributes, context); break; } case 'markup': { - end = context !== 'selector' ? markup?.(exp, context) : undefined; + end = markup?.(exp, context); handleOptions(exp.options, context); handleAttributes(exp.attributes, context); break; @@ -152,14 +151,13 @@ export function visit( for (const decl of msg.declarations) { const end = declaration?.(decl); if (decl.value) handleElement(decl.value, 'declaration'); - else for (const exp of decl.expressions) handleElement(exp, 'declaration'); end?.(); } if (msg.type === 'message') { handlePattern(msg.pattern); } else { - for (const sel of msg.selectors) handleElement(sel, 'selector'); + if (value) for (const sel of msg.selectors) value(sel, 'selector', 'arg'); for (const vari of msg.variants) { const end = variant?.(vari); if (key) vari.keys.forEach(key); diff --git a/packages/mf2-messageformat/src/errors.ts b/packages/mf2-messageformat/src/errors.ts index 5e76d87d..b0434f2c 100644 --- a/packages/mf2-messageformat/src/errors.ts +++ b/packages/mf2-messageformat/src/errors.ts @@ -88,9 +88,7 @@ export class MessageResolutionError extends MessageError { | 'bad-function-result' | 'bad-operand' | 'bad-option' - | 'unresolved-variable' - | 'unsupported-expression' - | 'unsupported-statement'; + | 'unresolved-variable'; source: string; constructor( type: typeof MessageResolutionError.prototype.type, diff --git a/packages/mf2-messageformat/src/functions/number.ts b/packages/mf2-messageformat/src/functions/number.ts index 0f90f73b..21f1d109 100644 --- a/packages/mf2-messageformat/src/functions/number.ts +++ b/packages/mf2-messageformat/src/functions/number.ts @@ -40,6 +40,8 @@ export interface MessageNumberPart extends MessageExpressionPart { parts: Intl.NumberFormatPart[]; } +const INT = Symbol('INT'); + /** * `number` accepts a number, BigInt or string representing a JSON number as input * and formats it with the same options as @@ -77,39 +79,41 @@ export function number( const msg = 'Input is not numeric'; throw new MessageResolutionError('bad-operand', msg, source); } - if (options) { - for (const [name, value] of Object.entries(options)) { - if (value === undefined) continue; - try { - switch (name) { - case 'locale': - case 'type': // used internally by Intl.PluralRules, but called 'select' here - break; - case 'minimumIntegerDigits': - case 'minimumFractionDigits': - case 'maximumFractionDigits': - case 'minimumSignificantDigits': - case 'maximumSignificantDigits': - case 'roundingIncrement': - // @ts-expect-error TS types don't know about roundingIncrement - opt[name] = asPositiveInteger(value); - break; - case 'useGrouping': - opt[name] = asBoolean(value); - break; - default: - // @ts-expect-error Unknown options will be ignored - opt[name] = asString(value); - } - } catch { - const msg = `Value ${value} is not valid for :number option ${name}`; - throw new MessageResolutionError('bad-option', msg, source); + for (const [name, optval] of Object.entries(options)) { + if (optval === undefined) continue; + try { + switch (name) { + case 'locale': + case 'type': // used internally by Intl.PluralRules, but called 'select' here + break; + case 'minimumIntegerDigits': + case 'minimumFractionDigits': + case 'maximumFractionDigits': + case 'minimumSignificantDigits': + case 'maximumSignificantDigits': + case 'roundingIncrement': + // @ts-expect-error TS types don't know about roundingIncrement + opt[name] = asPositiveInteger(optval); + break; + case 'useGrouping': + opt[name] = asBoolean(optval); + break; + default: + // @ts-expect-error Unknown options will be ignored + opt[name] = asString(optval); } + } catch { + const msg = `Value ${optval} is not valid for :number option ${name}`; + throw new MessageResolutionError('bad-option', msg, source); } } + const num = + Number.isFinite(value) && options[INT] + ? Math.round(value as number) + : value; + const lc = mergeLocales(locales, input, options); - const num = value; let locale: string | undefined; let nf: Intl.NumberFormat | undefined; let cat: Intl.LDMLPluralRule | undefined; @@ -167,6 +171,6 @@ export const integer = ( ) => number( ctx, - { ...options, maximumFractionDigits: 0, style: 'decimal' }, + { ...options, maximumFractionDigits: 0, style: 'decimal', [INT]: true }, input ); diff --git a/packages/mf2-messageformat/src/index.ts b/packages/mf2-messageformat/src/index.ts index 164b203a..61e35e8e 100644 --- a/packages/mf2-messageformat/src/index.ts +++ b/packages/mf2-messageformat/src/index.ts @@ -6,18 +6,17 @@ export type * from './formatted-parts.js'; export { parseCST } from './cst/parse-cst.js'; export { stringifyCST } from './cst/stringify-cst.js'; export { messageFromCST, cst } from './data-model/from-cst.js'; -export { type MessageParserOptions, parseMessage } from './data-model/parse.js'; +export { parseMessage } from './data-model/parse.js'; export { stringifyMessage } from './data-model/stringify.js'; export { isCatchallKey, isExpression, - isFunctionAnnotation, + isFunctionRef, isLiteral, isMarkup, isMessage, isPatternMessage, isSelectMessage, - isUnsupportedAnnotation, isVariableRef } from './data-model/type-guards.js'; export { validate } from './data-model/validate.js'; diff --git a/packages/mf2-messageformat/src/messageformat.ts b/packages/mf2-messageformat/src/messageformat.ts index 8ce7b94c..bdee506b 100644 --- a/packages/mf2-messageformat/src/messageformat.ts +++ b/packages/mf2-messageformat/src/messageformat.ts @@ -4,11 +4,7 @@ import { resolveExpression } from './data-model/resolve-expression.js'; import { UnresolvedExpression } from './data-model/resolve-variable.js'; import type { Message } from './data-model/types.js'; import { validate } from './data-model/validate.js'; -import { - MessageDataModelError, - MessageError, - MessageResolutionError -} from './errors.js'; +import { MessageDataModelError, MessageError } from './errors.js'; import type { Context } from './format-context.js'; import type { MessagePart } from './formatted-parts.js'; import { @@ -33,7 +29,7 @@ const defaultFunctions = Object.freeze({ }); /** - * The runtime function registry available when resolving {@link FunctionAnnotation} elements. + * The runtime function registry available when resolving {@link FunctionRef} elements. * * @beta */ @@ -173,21 +169,10 @@ export class MessageFormat { ) { const scope = { ...msgParams }; for (const decl of this.#message.declarations) { - switch (decl.type) { - case 'input': - scope[decl.name] = new UnresolvedExpression(decl.value, msgParams); - break; - case 'local': - scope[decl.name] = new UnresolvedExpression(decl.value); - break; - default: { - const source = decl.keyword ?? '�'; - const msg = `Reserved ${source} annotation is not supported`; - onError( - new MessageResolutionError('unsupported-statement', msg, source) - ); - } - } + scope[decl.name] = new UnresolvedExpression( + decl.value, + decl.type === 'input' ? msgParams ?? {} : undefined + ); } const ctx: Context = { onError, diff --git a/packages/mf2-messageformat/src/mf2-features.test.ts b/packages/mf2-messageformat/src/mf2-features.test.ts index f9a09254..e1c25883 100644 --- a/packages/mf2-messageformat/src/mf2-features.test.ts +++ b/packages/mf2-messageformat/src/mf2-features.test.ts @@ -1,5 +1,6 @@ import { fluentToResource } from '@messageformat/fluent'; import { + getMF1Functions, mf1ToMessage, mf1ToMessageData } from '@messageformat/icu-messageformat-1'; @@ -45,7 +46,7 @@ describe('Plural Range Selectors & Range Formatters (unicode-org/message-format- 'nl', source` .input {$range :range} - .match {$range} + .match $range one {{{$range} dag}} * {{{$range} dagen}} `, @@ -134,7 +135,7 @@ describe('Multi-selector messages (unicode-org/message-format-wg#119)', () => { expect(msg.selectors).toHaveLength(6); expect(msg.variants).toHaveLength(64); - const mf = new MessageFormat('en', msg); + const mf = new MessageFormat('en', msg, { functions: getMF1Functions() }); const one = mf.format({ N: 1, @@ -268,7 +269,8 @@ maybe('List formatting', () => { const mf = new MessageFormat( 'ro', source` - .match {$count :number} + .input {$count :number} + .match $count one {{I-am dat cadouri {$list :list each=dative}.}} * {{Le-am dat cadouri {$list :list each=dative}.}} `, diff --git a/packages/mf2-messageformat/src/select-pattern.ts b/packages/mf2-messageformat/src/select-pattern.ts index 5762b6ad..7a8adbde 100644 --- a/packages/mf2-messageformat/src/select-pattern.ts +++ b/packages/mf2-messageformat/src/select-pattern.ts @@ -1,4 +1,4 @@ -import { resolveExpression } from './data-model/resolve-expression.js'; +import { resolveVariableRef } from './data-model/resolve-variable.js'; import type { Message, Pattern } from './data-model/types.js'; import { MessageSelectionError } from './errors.js'; import type { Context } from './format-context.js'; @@ -10,7 +10,7 @@ export function selectPattern(context: Context, message: Message): Pattern { case 'select': { const ctx = message.selectors.map(sel => { - const selector = resolveExpression(context, sel); + const selector = resolveVariableRef(context, sel); let selectKey; if (typeof selector.selectKey === 'function') { selectKey = selector.selectKey.bind(selector); diff --git a/packages/mf2-messageformat/src/spec.test.ts b/packages/mf2-messageformat/src/spec.test.ts index 448a35c5..37360c1e 100644 --- a/packages/mf2-messageformat/src/spec.test.ts +++ b/packages/mf2-messageformat/src/spec.test.ts @@ -126,7 +126,6 @@ for (const scenario of testScenarios('test/messageformat-wg/test/tests')) { describe(scenario.scenario, () => { for (const tc of testCases(scenario)) { (tc.only ? describe.only : describe)(testName(tc), tests(tc)); - //if (tc.only) describe(testName(tc), tests(tc)); } }); } diff --git a/packages/mf2-xliff/src/mf2xliff.ts b/packages/mf2-xliff/src/mf2xliff.ts index c9fb90a6..721149d6 100644 --- a/packages/mf2-xliff/src/mf2xliff.ts +++ b/packages/mf2-xliff/src/mf2xliff.ts @@ -1,7 +1,6 @@ import deepEqual from 'fast-deep-equal'; import { MessageFormat, - isFunctionAnnotation, isLiteral, isMessage, isSelectMessage, @@ -13,7 +12,7 @@ import type * as X from './xliff-spec'; import { toNmtoken } from './nmtoken'; let _id = 0; -const nextId = () => `m${++_id}`; +const nextId = () => String(++_id); const star = Symbol('*'); @@ -103,19 +102,8 @@ function resolveMessage( if (isSelectMessage(srcMsg) || (trgMsg && isSelectMessage(trgMsg))) { return resolveSelect(key, srcMsg, trgMsg); } - const rdElements: X.ResourceItem[] = []; - const addRef = (elements: X.MessageElements) => { - const id = nextId(); - const ri: X.ResourceItem = { - type: 'element', - name: 'res:resourceItem', - attributes: { id }, - elements: [{ type: 'element', name: 'res:source', elements }] - }; - rdElements.push(ri); - return id; - }; - const segment = resolvePattern(srcMsg.pattern, trgMsg?.pattern, addRef); + const rdElements = resolveDeclarations(srcMsg, trgMsg); + const segment = resolvePattern(srcMsg.pattern, trgMsg?.pattern, rdElements); return buildUnit(key, rdElements, [segment]); } @@ -148,53 +136,34 @@ function resolveSelect( srcSel: MF.Message, trgSel: MF.Message | undefined ): X.Unit { + const rdElements = resolveDeclarations(srcSel, trgSel); + // We might be combining a Pattern and a Select, so let's normalise - if (isSelectMessage(srcSel)) { - if (trgSel && !isSelectMessage(trgSel)) { - trgSel = { - type: 'select', - declarations: [], - selectors: srcSel.selectors, - variants: [{ keys: [], value: trgSel.pattern }] - }; - } - } else { - if (!trgSel || !isSelectMessage(trgSel)) { - throw new Error( - `At least one of source & target at ${key.join('.')} must be a select` - ); - } + if (!isSelectMessage(srcSel)) { srcSel = { type: 'select', declarations: [], - selectors: trgSel.selectors, + selectors: [], variants: [{ keys: [], value: srcSel.pattern }] }; } - - const select: { id: string; keys: (string | typeof star)[] }[] = []; - const rdElements: X.ResourceItem[] = []; - const addRef = (elements: X.MessageElements) => { - const id = nextId(); - const ri: X.ResourceItem = { - type: 'element', - name: 'res:resourceItem', - attributes: { id }, - elements: [{ type: 'element', name: 'res:source', elements }] + if (trgSel && !isSelectMessage(trgSel)) { + trgSel = { + type: 'select', + declarations: [], + selectors: [], + variants: [{ keys: [], value: trgSel.pattern }] }; - rdElements.push(ri); - return id; - }; - for (const sel of srcSel.selectors) { - const id = addRef(resolveExpression(sel)); - select.push({ id, keys: [] }); } + + const select: { id: string; keys: (string | typeof star)[] }[] = + srcSel.selectors.map(sel => ({ id: sel.name, keys: [] })); const segments: X.Segment[] = []; if (!trgSel) { // If there's only a source, we use its cases directly for (const v of srcSel.variants) { - const segment = resolvePattern(v.value, undefined, addRef); + const segment = resolvePattern(v.value, undefined, rdElements); const vk = v.keys.map(k => (k.type === '*' ? star : k.value)); segment.attributes = { id: msgId('s', key, vk) }; segments.push(segment); @@ -204,13 +173,12 @@ function resolveSelect( // First, let's make sure that `selIds` and `parts` includes all the selectors // and that we have mappings between the array indices. const trgSelMap: number[] = []; - for (const sel of trgSel.selectors) { - const prevIdx = srcSel.selectors.findIndex(prev => deepEqual(sel, prev)); + for (const { name } of trgSel.selectors) { + const prevIdx = srcSel.selectors.findIndex(prev => prev.name === name); if (prevIdx !== -1) { trgSelMap.push(prevIdx); } else { - const id = addRef(resolveExpression(sel)); - select.push({ id, keys: [] }); + select.push({ id: name, keys: [] }); trgSelMap.push(select.length - 1); } } @@ -258,7 +226,7 @@ function resolveSelect( if (!srcCase || !trgCase) { throw new Error(`Case ${sk} not found‽ src:${srcCase} trg:${trgCase}`); } - const segment = resolvePattern(srcCase.value, trgCase.value, addRef); + const segment = resolvePattern(srcCase.value, trgCase.value, rdElements); segment.attributes = { id: msgId('s', key, sk) }; segments.push(segment); } @@ -270,6 +238,116 @@ function resolveSelect( return unit; } +function resolveDeclarations( + srcMsg: MF.Message, + trgMsg: MF.Message | undefined +): X.ResourceItem[] { + const rdElements: X.ResourceItem[] = srcMsg.declarations.map(decl => { + const elements = resolveExpression(decl.value!); + return { + type: 'element', + name: 'res:resourceItem', + attributes: { + id: decl.name!, + 'mf:declaration': decl.type as 'input' | 'local' + }, + elements: [{ type: 'element', name: 'res:source', elements }] + }; + }); + for (const decl of trgMsg?.declarations ?? []) { + const rt: X.ResourceTarget = { + type: 'element', + name: 'res:target', + elements: resolveExpression(decl.value!) + }; + const prev = rdElements.find(ri => ri.attributes.id === decl.name); + if (prev) { + if (prev.attributes['mf:declaration'] !== decl.type) { + throw new Error(`Cannot mix declaration types for $${decl.name}`); + } + prev.elements.push(rt); + } else { + rdElements.push({ + type: 'element', + name: 'res:resourceItem', + attributes: { + id: decl.name!, + 'mf:declaration': decl.type as 'input' | 'local' + }, + elements: [rt] + }); + } + } + return rdElements; +} + +function addRef( + rdElements: X.ResourceItem[], + kind: 'source' | 'target', + exp: MF.Markup +): { id: string; elements: [X.MessageMarkup, ...X.MessageAttribute[]] }; +function addRef( + rdElements: X.ResourceItem[], + kind: 'source' | 'target', + exp: MF.Expression | MF.Markup +): { id: string; elements: X.MessageElements }; +function addRef( + rdElements: X.ResourceItem[], + kind: 'source' | 'target', + exp: MF.Expression | MF.Markup +): { id: string; elements: X.MessageElements } { + const resName = kind === 'source' ? 'res:source' : 'res:target'; + + // For a bare variable reference, look for a matching declaration + if ( + exp.type === 'expression' && + exp.arg?.type === 'variable' && + !exp.functionRef + ) { + const name = exp.arg.name; + const prev = rdElements.find(ri => ri.attributes.id === name); + if (prev) { + const elements = (prev.elements.find(el => el.name === resName) + ?.elements ?? []) as X.MessageElements; + return { id: prev.attributes.id, elements }; + } + } + + const elements = + exp.type === 'expression' ? resolveExpression(exp) : resolveMarkup(exp); + + // Reuse previous references + const prev = rdElements.find(ri => + ri.elements.some( + el => el.name === resName && deepEqual(el.elements, elements) + ) + ); + if (prev) return { id: prev.attributes.id, elements }; + if (kind === 'target') { + const prevSource = rdElements.find( + ri => + ri.elements.length === 1 && + ri.elements[0].name === 'res:source' && + deepEqual(ri.elements[0].elements, elements) + ); + if (prevSource) { + prevSource.elements.push({ type: 'element', name: resName, elements }); + return { id: prevSource.attributes.id, elements }; + } + } + + // Add new reference with generated identifier + const id = `ph:${nextId()}`; + const ri: X.ResourceItem = { + type: 'element', + name: 'res:resourceItem', + attributes: { id }, + elements: [{ type: 'element', name: resName, elements }] + }; + rdElements.push(ri); + return { id, elements }; +} + function everyKey( select: { keys: (string | typeof star)[] }[] ): Iterable<(string | typeof star)[]> { @@ -296,10 +374,15 @@ function everyKey( function resolvePattern( srcPattern: MF.Pattern, trgPattern: MF.Pattern | undefined, - addRef: (elements: X.MessageElements) => string + rdElements: X.ResourceItem[] ): X.Segment { - const openMarkup: { id: string; markup: X.MessageMarkup }[] = []; + const openMarkup: { + id: string; + markup: X.MessageMarkup; + startRef: string; + }[] = []; const handlePart = ( + kind: 'source' | 'target', p: string | MF.Expression | MF.Markup ): X.Text | X.InlineElement => { if (typeof p === 'string') return asText(p); @@ -312,28 +395,36 @@ function resolvePattern( if (oi === -1) { isolated = 'yes'; } else { - const [{ id }] = openMarkup.splice(oi, 1); + const [{ id, startRef }] = openMarkup.splice(oi, 1); return { type: 'element', name: 'ec', - attributes: { startRef: id.substring(1), 'mf:ref': id } + attributes: { startRef, 'mf:ref': id } }; } } - const markup = resolveMarkup(p); - const id = addRef(markup); - openMarkup.unshift({ id, markup: markup[0] }); - return { - type: 'element', - name: p.kind === 'open' ? 'sc' : p.kind === 'close' ? 'ec' : 'ph', - attributes: { id: id.substring(1), isolated, 'mf:ref': id } - }; + const { id, elements } = addRef(rdElements, kind, p); + if (p.kind === 'open') { + const startRef = nextId(); + openMarkup.unshift({ id, markup: elements[0], startRef }); + return { + type: 'element', + name: 'sc', + attributes: { id: startRef, 'mf:ref': id } + }; + } else { + return { + type: 'element', + name: p.kind === 'close' ? 'ec' : 'ph', + attributes: { id: nextId(), isolated, 'mf:ref': id } + }; + } } - const id = addRef(resolveExpression(p)); + const { id } = addRef(rdElements, kind, p); return { type: 'element', name: 'ph', - attributes: { id: id.substring(1), 'mf:ref': id } + attributes: { id: nextId(), 'mf:ref': id } }; }; const cleanMarkupSpans = (elements: (X.Text | X.InlineElement)[]) => { @@ -357,12 +448,12 @@ function resolvePattern( } }; - const se = srcPattern.map(handlePart); + const se = srcPattern.map(part => handlePart('source', part)); cleanMarkupSpans(se); const source: X.Source = { type: 'element', name: 'source', elements: se }; let ge: X.Segment['elements']; if (trgPattern) { - const te = trgPattern.map(handlePart); + const te = trgPattern.map(part => handlePart('target', part)); cleanMarkupSpans(te); const target: X.Target = { type: 'element', name: 'target', elements: te }; ge = [source, target]; @@ -374,32 +465,24 @@ function resolvePattern( function resolveExpression({ arg, - annotation, + functionRef, attributes }: MF.Expression): X.MessageElements { - let resFunc: X.MessageFunction | X.MessageUnsupported | undefined; - if (annotation) { - if (isFunctionAnnotation(annotation)) { - const elements: X.MessageFunction['elements'] = []; - if (annotation.options) { - for (const [name, value] of annotation.options) { - elements.push({ - type: 'element', - name: 'mf:option', - attributes: { name }, - elements: [resolveArgument(value)] - }); - } + let resFunc: X.MessageFunction | undefined; + if (functionRef) { + const elements: X.MessageFunction['elements'] = []; + if (functionRef.options) { + for (const [name, value] of functionRef.options) { + elements.push({ + type: 'element', + name: 'mf:option', + attributes: { name }, + elements: [resolveArgument(value)] + }); } - const attributes = { name: annotation.name }; - resFunc = { type: 'element', name: 'mf:function', attributes, elements }; - } else { - resFunc = { - type: 'element', - name: 'mf:unsupported', - elements: [asText(annotation.source ?? '�')] - }; } + const attributes = { name: functionRef.name }; + resFunc = { type: 'element', name: 'mf:function', attributes, elements }; } let elements: X.MessageElements; diff --git a/packages/mf2-xliff/src/xliff-spec.ts b/packages/mf2-xliff/src/xliff-spec.ts index 73ee6216..c51c9062 100644 --- a/packages/mf2-xliff/src/xliff-spec.ts +++ b/packages/mf2-xliff/src/xliff-spec.ts @@ -1070,8 +1070,9 @@ export interface ResourceItem extends Element { name: 'res:resourceItem'; attributes: { context?: YesNo; - id?: string; + id: string; mimeType?: string; + 'mf:declaration'?: 'input' | 'local'; [key: string]: string | number | undefined; }; elements: (ResourceSource | ResourceTarget | ResourceReference)[]; @@ -1170,20 +1171,10 @@ export interface ValidationRule extends Element { export type MessageElements = | [ - ( - | MessageLiteral - | MessageVariable - | MessageMarkup - | MessageFunction - | MessageUnsupported - ), + MessageLiteral | MessageVariable | MessageMarkup | MessageFunction, ...MessageAttribute[] ] - | [ - MessageLiteral | MessageVariable, - MessageFunction | MessageUnsupported, - ...MessageAttribute[] - ]; + | [MessageLiteral | MessageVariable, MessageFunction, ...MessageAttribute[]]; export interface MessageMarkup extends Element { name: 'mf:markup'; @@ -1231,11 +1222,6 @@ export interface MessageOption extends Element { elements: [MessageLiteral | MessageVariable]; } -export interface MessageUnsupported extends Element { - name: 'mf:unsupported'; - elements: (Text | CharCode)[]; -} - export interface MessageVariable extends Element { name: 'mf:variable'; attributes: { diff --git a/packages/mf2-xliff/src/xliff.test.ts b/packages/mf2-xliff/src/xliff.test.ts index 60b8891f..561f91cb 100644 --- a/packages/mf2-xliff/src/xliff.test.ts +++ b/packages/mf2-xliff/src/xliff.test.ts @@ -1,21 +1,18 @@ import { fluentToResourceData } from '@messageformat/fluent'; import { source } from '@messageformat/test-utils'; import { mf2xliff, stringify, xliff2mf } from './index'; -import { - Message, - messageFromCST, - parseCST, - stringifyMessage -} from 'messageformat'; +import { Message, parseMessage, stringifyMessage } from 'messageformat'; test('source only', () => { const data = new Map([ - ['msg', messageFromCST(parseCST('Message'))], - ['var', messageFromCST(parseCST('Foo {$num}'))], - ['ref', messageFromCST(parseCST('This is the {msg :message @attr}'))], + ['msg', parseMessage('Message')], + ['var', parseMessage('Foo {$num}')], + ['ref', parseMessage('This is the {msg :message @attr}')], [ 'select', - messageFromCST(parseCST('.match {$selector :string} a {{A}} * {{B}}')) + parseMessage( + '.input {$selector :string} .match $selector a {{A}} * {{B}}' + ) ] ]); const xliff = stringify(mf2xliff({ data, id: 'res', locale: 'en' })); @@ -30,19 +27,19 @@ test('source only', () => { - + - Foo + Foo - + msg @@ -51,12 +48,12 @@ test('source only', () => { - This is the + This is the - + - + @@ -94,7 +91,11 @@ test('source only', () => { [['msg'], 'Message', undefined], [['var'], 'Foo {$num}', undefined], [['ref'], 'This is the {msg :message @attr}', undefined], - [['select'], '.match {$selector :string}\na {{A}}\n* {{B}}', undefined] + [ + ['select'], + '.input {$selector :string}\n.match $selector\na {{A}}\n* {{B}}', + undefined + ] ]); }); @@ -138,31 +139,33 @@ test('combine source & target', () => { - + - - - + - + - Foo - Föö + Foo + Föö - + - + + + + + @@ -190,8 +193,8 @@ test('combine source & target', () => { [['var'], 'Foo {$num}', 'Föö {$num}'], [ ['select'], - '.match {$selector :string}\na {{A}}\n* {{B}}', - '.match {$selector :string}\na {{Ä}}\n* {{B}}' + '.input {$selector :string}\n.match $selector\na {{A}}\n* {{B}}', + '.input {$selector :string}\n.match $selector\na {{Ä}}\n* {{B}}' ] ]); }); @@ -223,19 +226,19 @@ test('selector mismatch between source & target languages', () => { - + - + - - + + - + @@ -278,21 +281,17 @@ test('selector mismatch between source & target languages', () => { [ ['select'], source` - .match {$gender :string} {$case :string} - masculine allative {{his house}} - masculine * {{his house}} - feminine allative {{her house}} - feminine * {{her house}} - * allative {{their house}} - * * {{their house}}`, + .input {$gender :string} + .match $gender + masculine {{his house}} + feminine {{her house}} + * {{their house}}`, source` - .match {$gender :string} {$case :string} - masculine allative {{hänen talolle}} - masculine * {{hänen talo}} - feminine allative {{hänen talolle}} - feminine * {{hänen talo}} - * allative {{hänen talolle}} - * * {{hänen talo}}` + .input {$gender :string} + .input {$case :string} + .match $case + allative {{hänen talolle}} + * {{hänen talo}}` ] ]); }); @@ -346,7 +345,7 @@ describe('Parsing xml:space in parent elements', () => { - + msg @@ -354,7 +353,7 @@ describe('Parsing xml:space in parent elements', () => { - Message + Message @@ -372,9 +371,9 @@ describe('Parsing xml:space in parent elements', () => { const xliff = source` - + - + @@ -397,7 +396,11 @@ describe('Parsing xml:space in parent elements', () => { target && stringifyMessage(target) ]) ).toEqual([ - [['key'], '.match {$sel :string}\na {{ A }}\n* {{ B }}', undefined] + [ + ['key'], + '.input {$sel :string}\n.match $sel\na {{ A }}\n* {{ B }}', + undefined + ] ]); }); }); @@ -432,13 +435,17 @@ test('variably available targets', () => { - + - + + + + + @@ -469,12 +476,14 @@ test('variably available targets', () => { [ ['five'], source` - .match {$x :number} + .input {$x :number} + .match $x 0 {{A}} one {{B}} * {{C}}`, source` - .match {$x :number} + .input {$x :number} + .match $x 0 {{Ä}} * {{C}}` ] diff --git a/packages/mf2-xliff/src/xliff2mf.ts b/packages/mf2-xliff/src/xliff2mf.ts index c0cd2801..3c1c2c92 100644 --- a/packages/mf2-xliff/src/xliff2mf.ts +++ b/packages/mf2-xliff/src/xliff2mf.ts @@ -1,3 +1,4 @@ +import deepEqual from 'fast-deep-equal'; import type * as MF from 'messageformat'; import type * as X from './xliff-spec'; import { parse } from './xliff'; @@ -97,13 +98,13 @@ function resolveSelectMessage( } const source: MF.SelectMessage = { type: 'select', - declarations: [], + declarations: resolveDeclarations('source', rd), selectors: [], variants: [] }; const target: MF.SelectMessage = { type: 'select', - declarations: [], + declarations: resolveDeclarations('target', rd), selectors: [], variants: [] }; @@ -112,7 +113,9 @@ function resolveSelectMessage( const keys = parseId('s', el.attributes?.id).variant; const pattern = resolvePattern(rd, el); source.variants.push({ keys, value: pattern.source }); - if (pattern.target) target.variants.push({ keys, value: pattern.target }); + if (pattern.target) { + target.variants.push({ keys: keys.slice(), value: pattern.target }); + } } } if (!source.variants.length) { @@ -120,14 +123,62 @@ function resolveSelectMessage( throw new Error(`No variant elements found in ${el}`); } const hasTarget = !!target.variants.length; + const srcSelectors: boolean[] = []; + const tgtSelectors: boolean[] = []; for (const ref of attributes['mf:select']!.trim().split(/\s+/)) { - const srcElements = getMessageElements('source', rd!, ref); - source.selectors.push(resolveExpression(srcElements)); - if (hasTarget) { - const tgtElements = getMessageElements('target', rd!, ref); - target.selectors.push(resolveExpression(tgtElements)); + const ri = rd.elements.find( + ri => + ri.name === 'res:resourceItem' && + ri.attributes?.id === ref && + ri.attributes['mf:declaration'] + ) as X.ResourceItem | undefined; + if (!ri) throw new Error(`Unresolved MessageFormat reference: ${ref}`); + let srcSel = false; + let tgtSel = false; + for (const el of ri.elements) { + if (el.name === 'res:source') { + source.selectors.push({ type: 'variable', name: ref }); + srcSel = true; + } else if (hasTarget && el.name === 'res:target') { + target.selectors.push({ type: 'variable', name: ref }); + tgtSel = true; + } + } + srcSelectors.push(srcSel); + tgtSelectors.push(tgtSel); + } + + if (srcSelectors.some(s => !s)) { + for (const { keys } of source.variants) { + for (let i = srcSelectors.length - 1; i >= 0; --i) { + if (!srcSelectors[i]) keys.splice(i, 1); + } + } + for (let i = 0; i < source.variants.length - 1; ++i) { + const { keys } = source.variants[i]; + for (let j = source.variants.length - 1; j > i; --j) { + if (deepEqual(keys, source.variants[j].keys)) { + source.variants.splice(j, 1); + } + } } } + if (tgtSelectors.some(s => !s)) { + for (const { keys } of target.variants) { + for (let i = tgtSelectors.length - 1; i >= 0; --i) { + if (!tgtSelectors[i]) keys.splice(i, 1); + } + } + for (let i = 0; i < target.variants.length - 1; ++i) { + const { keys } = target.variants[i]; + for (let j = target.variants.length - 1; j > i; --j) { + if (deepEqual(keys, target.variants[j].keys)) { + target.variants.splice(j, 1); + } + } + } + } + return { source, target: hasTarget ? target : undefined }; } @@ -155,13 +206,44 @@ function resolvePatternMessage( } } return { - source: { type: 'message', declarations: [], pattern: source }, + source: { + type: 'message', + declarations: resolveDeclarations('source', rd), + pattern: source + }, target: hasTarget - ? { type: 'message', declarations: [], pattern: target } + ? { + type: 'message', + declarations: resolveDeclarations('target', rd), + pattern: target + } : undefined }; } +function resolveDeclarations( + st: 'source' | 'target', + rd: X.ResourceData | undefined +): MF.Declaration[] { + const declarations: MF.Declaration[] = []; + for (const ri of rd?.elements ?? []) { + if (ri.name === 'res:resourceItem' && ri.attributes['mf:declaration']) { + const mfElements = getMessageElements(st, ri); + if (mfElements.length) { + const type = ri.attributes['mf:declaration']; + const name = ri.attributes.id; + const exp = resolveExpression(mfElements); + if (type === 'input' && exp.arg?.type !== 'variable') { + throw new Error(`Invalid .input declaration: ${name}`); + } + const value = exp as MF.Expression; + declarations.push({ type, name, value }); + } + } + } + return declarations; +} + function resolvePattern( rd: X.ResourceData | undefined, { name, attributes, elements }: X.Segment | X.Ignorable @@ -258,40 +340,47 @@ function resolvePlaceholder( `Resolving ${el} requires in the same ` ); } - const mfElements = getMessageElements(st, rd, String(ref)); + const ri = rd.elements.find( + ri => ri.name === 'res:resourceItem' && ri.attributes?.id === ref + ) as X.ResourceItem | undefined; + + if (ri?.attributes['mf:declaration']) { + return { type: 'expression', arg: { type: 'variable', name: String(ref) } }; + } + + const mfElements = getMessageElements(st, ri); + if (!mfElements.length) { + throw new Error(`Unresolved MessageFormat reference: ${ref}`); + } + if (mfElements[0].name === 'mf:markup') { const kind = name === 'ph' ? 'standalone' : name === 'ec' ? 'close' : 'open'; return resolveMarkup(mfElements[0], kind); } - if (name !== 'ph') { - throw new Error('Only elements may refer to expression values'); - } - return resolveExpression(mfElements); + if (name === 'ph') return resolveExpression(mfElements); + throw new Error(`Only elements may refer to expression values (${ref})`); } function getMessageElements( st: 'source' | 'target', - rd: X.ResourceData, - ref: string + ri: X.ResourceItem | undefined ) { - const ri = rd.elements.find(el => el.attributes?.id === ref); - if (ri?.elements) { - const parent = - (st === 'target' - ? ri.elements.find(el => el.name === 'res:target') - : null) ?? ri.elements.find(el => el.name === 'res:source'); - const mfElements = parent?.elements?.filter( - el => el.type === 'element' && el.name.startsWith('mf:') - ); - if (mfElements?.length) return mfElements as X.MessageElements; + let src: X.ResourceSource | undefined; + let tgt: X.ResourceTarget | undefined; + for (const el of ri?.elements ?? []) { + if (el.name === 'res:source') src = el; + else if (el.name === 'res:target') tgt = el; } - throw new Error(`Unresolved MessageFormat reference: ${ref}`); + const parent = st === 'target' ? tgt ?? src : src; + return (parent?.elements?.filter( + el => el.type === 'element' && el.name.startsWith('mf:') + ) ?? []) as X.MessageElements; } function resolveExpression(elements: X.MessageElements): MF.Expression { let xArg: X.MessageLiteral | X.MessageVariable | undefined; - let xFunc: X.MessageFunction | X.MessageUnsupported | undefined; + let xFunc: X.MessageFunction | undefined; const attributes: MF.Attributes = new Map(); for (const el of elements) { switch (el.name) { @@ -301,8 +390,7 @@ function resolveExpression(elements: X.MessageElements): MF.Expression { xArg = el; break; case 'mf:function': - case 'mf:unsupported': - if (xFunc) throw new Error('More than one annotation in an expression'); + if (xFunc) throw new Error('More than one function in an expression'); xFunc = el; break; case 'mf:markup': @@ -333,23 +421,15 @@ function resolveExpression(elements: X.MessageElements): MF.Expression { return { type: 'expression', arg, attributes }; } - let annotation: MF.FunctionAnnotation | MF.UnsupportedAnnotation; - if (xFunc.name === 'mf:function') { - annotation = { - type: 'function', - name: xFunc.attributes.name, - options: resolveOptions(xFunc) - }; - } else { - annotation = { - type: 'unsupported-annotation', - source: resolveText(xFunc.elements) - }; - } + const functionRef: MF.FunctionRef = { + type: 'function', + name: xFunc.attributes.name, + options: resolveOptions(xFunc) + }; return arg - ? { type: 'expression', arg, annotation, attributes } - : { type: 'expression', annotation, attributes }; + ? { type: 'expression', arg, functionRef, attributes } + : { type: 'expression', functionRef, attributes }; } function resolveMarkup( diff --git a/test/messageformat-wg b/test/messageformat-wg index 3eb6c86e..869005f9 160000 --- a/test/messageformat-wg +++ b/test/messageformat-wg @@ -1 +1 @@ -Subproject commit 3eb6c86e7c86dc191d00e02d53909cc9245e1592 +Subproject commit 869005f9ec4f44dd683cb0ac733c8fe7033c57dc