From ba27e4c4e2c827f74fd283a380373729dfb2f015 Mon Sep 17 00:00:00 2001 From: Augustine Kim Date: Mon, 29 Jan 2024 12:04:13 -0800 Subject: [PATCH 1/7] Make validation no longer error on changed template expression --- packages/localize-tools/src/messages.ts | 49 +++++++++++++++---- .../src/tests/e2e/placeholder-errors.test.ts | 6 +-- .../placeholder-errors/goldens/foo.ts | 2 + .../testdata/placeholder-errors/input/foo.ts | 2 + 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/packages/localize-tools/src/messages.ts b/packages/localize-tools/src/messages.ts index a2b71397cc..9ccfbd534b 100644 --- a/packages/localize-tools/src/messages.ts +++ b/packages/localize-tools/src/messages.ts @@ -4,8 +4,9 @@ * SPDX-License-Identifier: BSD-3-Clause */ -import * as ts from 'typescript'; +import ts from 'typescript'; import type {Locale} from './types/locale.js'; +import {parseStringAsTemplateLiteral} from './typescript.js'; /** * A message for translation. @@ -111,17 +112,20 @@ export function sortProgramMessages( * (no more, no less, no changes, but order can change). * * It is important to validate this condition because placeholders can contain - * arbitrary HTML and JavaScript template literal placeholder expressions, will - * be substituted back into generated executable source code. A well behaving - * localization process/tool would not allow any modification of these - * placeholders, but we can't assume that to be the case, so it is a potential - * source of bugs and attacks and must be validated. + * arbitrary HTML which will be substituted back into generated executable + * source code. A well behaving localization process/tool would not allow any + * modification of these placeholders, but we can't assume that to be the case, + * so it is a potential source of bugs and attacks and must be validated. + * + * JavaScript template expressions within placeholders are not validated since + * they are replaced by numbers in runtime mode, or the expression from the + * source code in transform mode. */ export function validateLocalizedPlaceholders( programMessages: Message[], localizedMessages: Map ): string[] { - const errors = []; + const errors: string[] = []; const programMap = makeMessageIdMap(programMessages); for (const [locale, messages] of localizedMessages) { for (const localizedMsg of messages) { @@ -138,13 +142,17 @@ export function validateLocalizedPlaceholders( const remainingProgramPlaceholders = []; for (const content of programMsg.contents) { if (typeof content !== 'string') { - remainingProgramPlaceholders.push(content.untranslatable); + remainingProgramPlaceholders.push( + replaceExpressionInTemplateString(content.untranslatable) + ); } } for (const content of localizedMsg.contents) { if (typeof content !== 'string') { - const placeholder = content.untranslatable; + const placeholder = replaceExpressionInTemplateString( + content.untranslatable + ); const index = remainingProgramPlaceholders.indexOf(placeholder); if (index === -1) { errors.push( @@ -169,3 +177,26 @@ export function validateLocalizedPlaceholders( } return errors; } + +/** + * Given a template string, replace all expression with a provided string (or + * "expr" if none provided). + * + * e.g. `hello ${foo} world ${bar}` -> `hello ${expr} world ${expr}` + */ +function replaceExpressionInTemplateString( + templateString: string, + expression = 'expr' +): string { + const template = parseStringAsTemplateLiteral(templateString); + if (ts.isNoSubstitutionTemplateLiteral(template)) { + return template.text; + } + const fragments: string[] = []; + fragments.push(template.head.text); + for (let i = 0; i < template.templateSpans.length; i++) { + fragments.push('${' + expression + '}'); + fragments.push(template.templateSpans[i].literal.text); + } + return fragments.join(''); +} diff --git a/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts b/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts index 6d86548ed2..7b5d9b0b8d 100644 --- a/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts +++ b/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts @@ -12,10 +12,8 @@ e2eGoldensTest( 1, `One or more localized templates contain a set of placeholders (HTML or template literal expressions) that do not exactly match the source code, aborting. Details: -Placeholder error in es-419 localization of extra-expression: unexpected "\${alert("evil")}" -Placeholder error in es-419 localization of missing-expression: missing "\${name}" -Placeholder error in es-419 localization of changed-expression: unexpected "\${alert("evil") || name}" -Placeholder error in es-419 localization of changed-expression: missing "\${name}" +Placeholder error in es-419 localization of extra-expression: unexpected "\${expr}" +Placeholder error in es-419 localization of missing-expression: missing "\${expr}" Placeholder error in es-419 localization of missing-html: missing "" Placeholder error in es-419 localization of missing-html: missing "" Placeholder error in es-419 localization of changed-html: unexpected "" diff --git a/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts b/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts index bcaa09572c..0116694706 100644 --- a/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts +++ b/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts @@ -11,6 +11,8 @@ const name = 'Friend'; msg(`Hello World`, {id: 'extra-expression'}); msg(str`Hello ${name}`, {id: 'missing-expression'}); +// Changed expression used to be considered errors but no longer are +// See https://github.com/lit/lit/issues/4502 msg(str`Hello ${name}`, {id: 'changed-expression'}); msg(html`Hello World`, {id: 'missing-html'}); msg(html`Hello World`, {id: 'changed-html'}); diff --git a/packages/localize-tools/testdata/placeholder-errors/input/foo.ts b/packages/localize-tools/testdata/placeholder-errors/input/foo.ts index bcaa09572c..0116694706 100644 --- a/packages/localize-tools/testdata/placeholder-errors/input/foo.ts +++ b/packages/localize-tools/testdata/placeholder-errors/input/foo.ts @@ -11,6 +11,8 @@ const name = 'Friend'; msg(`Hello World`, {id: 'extra-expression'}); msg(str`Hello ${name}`, {id: 'missing-expression'}); +// Changed expression used to be considered errors but no longer are +// See https://github.com/lit/lit/issues/4502 msg(str`Hello ${name}`, {id: 'changed-expression'}); msg(html`Hello World`, {id: 'missing-html'}); msg(html`Hello World`, {id: 'changed-html'}); From 2e6d262caafdc40560a12881f1bfd20155815fd0 Mon Sep 17 00:00:00 2001 From: Augustine Kim Date: Mon, 29 Jan 2024 16:35:02 -0800 Subject: [PATCH 2/7] Add e2e test for build-runtime-xliff --- .../testdata/build-runtime-xliff/goldens/foo.ts | 4 ++++ .../testdata/build-runtime-xliff/goldens/tsout/es-419.ts | 2 ++ .../testdata/build-runtime-xliff/goldens/tsout/zh_CN.ts | 2 ++ .../testdata/build-runtime-xliff/goldens/xliff/es-419.xlf | 8 ++++++++ .../testdata/build-runtime-xliff/goldens/xliff/zh_CN.xlf | 6 ++++++ .../testdata/build-runtime-xliff/input/foo.ts | 4 ++++ .../testdata/build-runtime-xliff/input/xliff/es-419.xlf | 8 ++++++++ .../testdata/build-runtime-xliff/input/xliff/zh_CN.xlf | 6 ++++++ 8 files changed, 40 insertions(+) diff --git a/packages/localize-tools/testdata/build-runtime-xliff/goldens/foo.ts b/packages/localize-tools/testdata/build-runtime-xliff/goldens/foo.ts index 14942b5406..3970c41824 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/goldens/foo.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff/goldens/foo.ts @@ -58,3 +58,7 @@ msg(html`Hello! Click here!`); // Escaped markup characters should remain escaped msg(html`<Hello<World & Friends>!>`); + +// Placeholder in translation has different expression +msg(str`Different ${user}`); +msg(html`Different ${user}`); diff --git a/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/es-419.ts b/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/es-419.ts index 7cfe75b33e..41b7bd24cd 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/es-419.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/es-419.ts @@ -11,6 +11,7 @@ export const templates = { h02c268d9b1fcb031: html`<Hola<Mundo & Amigos>!>`, h349c3c4777670217: html`[SALT] Hola ${0}!`, h3c44aff2d5f5ef6b: html`Hola Mundo!`, + h5e2d21ff71e6c8b5: html`${0} diferente`, h82ccc38d4d46eaa9: html`Hola ${0}!`, h8d70dfec810d1eae: html`Hola! Clic aquí!`, h99e74f744fda7e25: html`Clic aquí!`, @@ -21,5 +22,6 @@ export const templates = { s00ad08ebae1e0f74: str`Hola ${0}!`, s03c68d79ad36e8d4: `described 0`, s0f19e6c4e521dd53: `Mundo`, + s372f95c2b25986c8: str`${0} diferente`, s8c0ec8d1fb9e6e32: `Hola Mundo!`, }; diff --git a/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/zh_CN.ts b/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/zh_CN.ts index f30f0358c8..c8e05584e3 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/zh_CN.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff/goldens/tsout/zh_CN.ts @@ -22,4 +22,6 @@ export const templates = { s03c68d79ad36e8d4: `described 0`, h8d70dfec810d1eae: html`Hello! Click here!`, h02c268d9b1fcb031: html`<Hello<World & Friends>!>`, + s372f95c2b25986c8: str`Different ${0}`, + h5e2d21ff71e6c8b5: html`Different ${0}`, }; diff --git a/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/es-419.xlf b/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/es-419.xlf index 167122fd4a..c70d03b0d2 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/es-419.xlf @@ -59,6 +59,14 @@ described 0 described 0 + + Different + diferente + + + Different + diferente + diff --git a/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/zh_CN.xlf b/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/zh_CN.xlf index 6c2095f11e..f711e6d017 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/zh_CN.xlf +++ b/packages/localize-tools/testdata/build-runtime-xliff/goldens/xliff/zh_CN.xlf @@ -47,6 +47,12 @@ Description of 0 described 0 + + Different + + + Different + diff --git a/packages/localize-tools/testdata/build-runtime-xliff/input/foo.ts b/packages/localize-tools/testdata/build-runtime-xliff/input/foo.ts index 9a57654b0d..0075c47765 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/input/foo.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff/input/foo.ts @@ -58,3 +58,7 @@ msg(html`Hello! Click here!`); // Escaped markup characters should remain escaped msg(html`<Hello<World & Friends>!>`); + +// Placeholder in translation has different expression +msg(str`Different ${user}`); +msg(html`Different ${user}`); diff --git a/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/es-419.xlf b/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/es-419.xlf index 167122fd4a..c70d03b0d2 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/es-419.xlf @@ -59,6 +59,14 @@ described 0 described 0 + + Different + diferente + + + Different + diferente + diff --git a/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/zh_CN.xlf b/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/zh_CN.xlf index 6c2095f11e..f711e6d017 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/zh_CN.xlf +++ b/packages/localize-tools/testdata/build-runtime-xliff/input/xliff/zh_CN.xlf @@ -47,6 +47,12 @@ Description of 0 described 0 + + Different + + + Different + From 362bbb3df0ed7ead6494c388992ad806f674b337 Mon Sep 17 00:00:00 2001 From: Augustine Kim Date: Mon, 29 Jan 2024 16:38:03 -0800 Subject: [PATCH 3/7] Add e2e test for build-runtime-xliff-ph --- .../testdata/build-runtime-xliff-ph/goldens/foo.ts | 4 ++++ .../build-runtime-xliff-ph/goldens/tsout/es-419.ts | 2 ++ .../build-runtime-xliff-ph/goldens/tsout/zh_CN.ts | 2 ++ .../build-runtime-xliff-ph/goldens/xliff/es-419.xlf | 8 ++++++++ .../testdata/build-runtime-xliff-ph/input/foo.ts | 4 ++++ .../build-runtime-xliff-ph/input/xliff/es-419.xlf | 8 ++++++++ 6 files changed, 28 insertions(+) diff --git a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/foo.ts b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/foo.ts index 14942b5406..3970c41824 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/foo.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/foo.ts @@ -58,3 +58,7 @@ msg(html`Hello! Click here!`); // Escaped markup characters should remain escaped msg(html`<Hello<World & Friends>!>`); + +// Placeholder in translation has different expression +msg(str`Different ${user}`); +msg(html`Different ${user}`); diff --git a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/es-419.ts b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/es-419.ts index 7cfe75b33e..41b7bd24cd 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/es-419.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/es-419.ts @@ -11,6 +11,7 @@ export const templates = { h02c268d9b1fcb031: html`<Hola<Mundo & Amigos>!>`, h349c3c4777670217: html`[SALT] Hola ${0}!`, h3c44aff2d5f5ef6b: html`Hola Mundo!`, + h5e2d21ff71e6c8b5: html`${0} diferente`, h82ccc38d4d46eaa9: html`Hola ${0}!`, h8d70dfec810d1eae: html`Hola! Clic aquí!`, h99e74f744fda7e25: html`Clic aquí!`, @@ -21,5 +22,6 @@ export const templates = { s00ad08ebae1e0f74: str`Hola ${0}!`, s03c68d79ad36e8d4: `described 0`, s0f19e6c4e521dd53: `Mundo`, + s372f95c2b25986c8: str`${0} diferente`, s8c0ec8d1fb9e6e32: `Hola Mundo!`, }; diff --git a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/zh_CN.ts b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/zh_CN.ts index f30f0358c8..c8e05584e3 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/zh_CN.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/tsout/zh_CN.ts @@ -22,4 +22,6 @@ export const templates = { s03c68d79ad36e8d4: `described 0`, h8d70dfec810d1eae: html`Hello! Click here!`, h02c268d9b1fcb031: html`<Hello<World & Friends>!>`, + s372f95c2b25986c8: str`Different ${0}`, + h5e2d21ff71e6c8b5: html`Different ${0}`, }; diff --git a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/xliff/es-419.xlf b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/xliff/es-419.xlf index 8da3149daf..93cb2ea66b 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/build-runtime-xliff-ph/goldens/xliff/es-419.xlf @@ -59,6 +59,14 @@ <Hello<b><World & Friends></b>!> <Hola<b><Mundo & Amigos></b>!> + + Different ${user} + ${differentUser} diferente + + + Different <b>${user}</b> + <b>${differentUser}</b> diferente + diff --git a/packages/localize-tools/testdata/build-runtime-xliff-ph/input/foo.ts b/packages/localize-tools/testdata/build-runtime-xliff-ph/input/foo.ts index 9a57654b0d..0075c47765 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff-ph/input/foo.ts +++ b/packages/localize-tools/testdata/build-runtime-xliff-ph/input/foo.ts @@ -58,3 +58,7 @@ msg(html`Hello! Click here!`); // Escaped markup characters should remain escaped msg(html`<Hello<World & Friends>!>`); + +// Placeholder in translation has different expression +msg(str`Different ${user}`); +msg(html`Different ${user}`); diff --git a/packages/localize-tools/testdata/build-runtime-xliff-ph/input/xliff/es-419.xlf b/packages/localize-tools/testdata/build-runtime-xliff-ph/input/xliff/es-419.xlf index 8da3149daf..93cb2ea66b 100644 --- a/packages/localize-tools/testdata/build-runtime-xliff-ph/input/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/build-runtime-xliff-ph/input/xliff/es-419.xlf @@ -59,6 +59,14 @@ <Hello<b><World & Friends></b>!> <Hola<b><Mundo & Amigos></b>!> + + Different ${user} + ${differentUser} diferente + + + Different <b>${user}</b> + <b>${differentUser}</b> diferente + From c71665f672fc8d57feca8664c4ed0fcdad65e5f2 Mon Sep 17 00:00:00 2001 From: Augustine Kim Date: Mon, 29 Jan 2024 17:11:49 -0800 Subject: [PATCH 4/7] Add e2e tests for build-transform-xliff --- .../testdata/build-transform-xliff/goldens/foo.ts | 4 ++++ .../build-transform-xliff/goldens/tsout/en/foo.js | 3 +++ .../goldens/tsout/es-419/foo.js | 5 ++++- .../build-transform-xliff/goldens/tsout/zh_CN/foo.js | 3 +++ .../build-transform-xliff/goldens/xliff/es-419.xlf | 12 ++++++++++++ .../testdata/build-transform-xliff/input/foo.ts | 4 ++++ .../build-transform-xliff/input/xliff/es-419.xlf | 12 ++++++++++++ 7 files changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/localize-tools/testdata/build-transform-xliff/goldens/foo.ts b/packages/localize-tools/testdata/build-transform-xliff/goldens/foo.ts index 42aa97e684..1dea2f6357 100644 --- a/packages/localize-tools/testdata/build-transform-xliff/goldens/foo.ts +++ b/packages/localize-tools/testdata/build-transform-xliff/goldens/foo.ts @@ -86,3 +86,7 @@ html`Hello World`; html`HelloWorld`; html`Hello World`; html`Hello World`; + +// Placeholder in translation has different expression +msg(str`Different ${user}`); +msg(html`Different ${user}`); diff --git a/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/en/foo.js b/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/en/foo.js index a35b8c0c39..aee771354e 100644 --- a/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/en/foo.js +++ b/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/en/foo.js @@ -60,3 +60,6 @@ html`Hello World`; html`HelloWorld`; html`Hello World`; html`Hello World`; +// Placeholder in translation has different expression +`Different ${user}`; +html`Different ${user}`; diff --git a/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/es-419/foo.js b/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/es-419/foo.js index 5dd00a24e6..88a71b76e4 100644 --- a/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/es-419/foo.js +++ b/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/es-419/foo.js @@ -57,6 +57,9 @@ html`<Hola<Mundo & Amigos>!>`; // Expressions as attribute values should stay as expressions html`Hello World`; html`Hello World`; -html`HelloWorld`; +html`HelloWorld`; html`Hello World`; html`Hello World`; +// Placeholder in translation has different expression +`${user} diferente`; +html`${user} diferente`; diff --git a/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/zh_CN/foo.js b/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/zh_CN/foo.js index 1d66a5b375..4cf203a987 100644 --- a/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/zh_CN/foo.js +++ b/packages/localize-tools/testdata/build-transform-xliff/goldens/tsout/zh_CN/foo.js @@ -60,3 +60,6 @@ html`Hello World`; html`HelloWorld`; html`Hello World`; html`Hello World`; +// Placeholder in translation has different expression +`Different ${user}`; +html`Different ${user}`; diff --git a/packages/localize-tools/testdata/build-transform-xliff/goldens/xliff/es-419.xlf b/packages/localize-tools/testdata/build-transform-xliff/goldens/xliff/es-419.xlf index 1ec5dec94c..9204b83e83 100644 --- a/packages/localize-tools/testdata/build-transform-xliff/goldens/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/build-transform-xliff/goldens/xliff/es-419.xlf @@ -46,6 +46,18 @@ <Hello<b><World & Friends></b>!> <Hola<b><Mundo & Amigos></b>!> + + Hello + Hola + + + Different ${user} + ${differentUser} diferente + + + Different <b>${user}</b> + <b>${differentUser}</b> diferente + diff --git a/packages/localize-tools/testdata/build-transform-xliff/input/foo.ts b/packages/localize-tools/testdata/build-transform-xliff/input/foo.ts index 458826daec..cd8e6c8ac0 100644 --- a/packages/localize-tools/testdata/build-transform-xliff/input/foo.ts +++ b/packages/localize-tools/testdata/build-transform-xliff/input/foo.ts @@ -86,3 +86,7 @@ html`Hello World`; html`HelloWorld`; html`Hello World`; html`Hello World`; + +// Placeholder in translation has different expression +msg(str`Different ${user}`); +msg(html`Different ${user}`); diff --git a/packages/localize-tools/testdata/build-transform-xliff/input/xliff/es-419.xlf b/packages/localize-tools/testdata/build-transform-xliff/input/xliff/es-419.xlf index 1ec5dec94c..9204b83e83 100644 --- a/packages/localize-tools/testdata/build-transform-xliff/input/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/build-transform-xliff/input/xliff/es-419.xlf @@ -46,6 +46,18 @@ <Hello<b><World & Friends></b>!> <Hola<b><Mundo & Amigos></b>!> + + Hello + Hola + + + Different ${user} + ${differentUser} diferente + + + Different <b>${user}</b> + <b>${differentUser}</b> diferente + From fb6be1cd8cf27442059eea7806d2d5e22290e9e7 Mon Sep 17 00:00:00 2001 From: Augustine Kim Date: Mon, 29 Jan 2024 17:23:05 -0800 Subject: [PATCH 5/7] Add changeset --- .changeset/mighty-cars-confess.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/mighty-cars-confess.md diff --git a/.changeset/mighty-cars-confess.md b/.changeset/mighty-cars-confess.md new file mode 100644 index 0000000000..631586b69a --- /dev/null +++ b/.changeset/mighty-cars-confess.md @@ -0,0 +1,5 @@ +--- +'@lit/localize-tools': patch +--- + +Translated message validation that runs before the `build` step now disregards template literal expressions. This allow source code to have variables in expressions renamed while still keeping the same translations, or avoid errors that could happen from module import order changing which expression gets picked up first when multiple `msg()` calls with the same id have different expressions. This behavior is more consistent with how a translation unit is identified according to [how the message id is generated](https://lit.dev/docs/localization/overview/#id-generation). From 37dba2f97a2f9f4ba73f725fe47525d4cac7f804 Mon Sep 17 00:00:00 2001 From: Augustine Kim Date: Mon, 29 Jan 2024 18:17:28 -0800 Subject: [PATCH 6/7] Remove changed expression test from placeholder-errors --- .../localize-tools/testdata/placeholder-errors/goldens/foo.ts | 3 --- .../testdata/placeholder-errors/goldens/xliff/es-419.xlf | 4 ---- .../localize-tools/testdata/placeholder-errors/input/foo.ts | 3 --- .../testdata/placeholder-errors/input/xliff/es-419.xlf | 4 ---- 4 files changed, 14 deletions(-) diff --git a/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts b/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts index 0116694706..5cb4e1ad94 100644 --- a/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts +++ b/packages/localize-tools/testdata/placeholder-errors/goldens/foo.ts @@ -11,8 +11,5 @@ const name = 'Friend'; msg(`Hello World`, {id: 'extra-expression'}); msg(str`Hello ${name}`, {id: 'missing-expression'}); -// Changed expression used to be considered errors but no longer are -// See https://github.com/lit/lit/issues/4502 -msg(str`Hello ${name}`, {id: 'changed-expression'}); msg(html`Hello World`, {id: 'missing-html'}); msg(html`Hello World`, {id: 'changed-html'}); diff --git a/packages/localize-tools/testdata/placeholder-errors/goldens/xliff/es-419.xlf b/packages/localize-tools/testdata/placeholder-errors/goldens/xliff/es-419.xlf index 5be2a8984a..3c06709a9e 100644 --- a/packages/localize-tools/testdata/placeholder-errors/goldens/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/placeholder-errors/goldens/xliff/es-419.xlf @@ -10,10 +10,6 @@ Hello ${name} Hola Mundo - - Hello ${name} - Hola ${alert("evil") || name} - <b>Hello World</b> Hola Mundo diff --git a/packages/localize-tools/testdata/placeholder-errors/input/foo.ts b/packages/localize-tools/testdata/placeholder-errors/input/foo.ts index 0116694706..5cb4e1ad94 100644 --- a/packages/localize-tools/testdata/placeholder-errors/input/foo.ts +++ b/packages/localize-tools/testdata/placeholder-errors/input/foo.ts @@ -11,8 +11,5 @@ const name = 'Friend'; msg(`Hello World`, {id: 'extra-expression'}); msg(str`Hello ${name}`, {id: 'missing-expression'}); -// Changed expression used to be considered errors but no longer are -// See https://github.com/lit/lit/issues/4502 -msg(str`Hello ${name}`, {id: 'changed-expression'}); msg(html`Hello World`, {id: 'missing-html'}); msg(html`Hello World`, {id: 'changed-html'}); diff --git a/packages/localize-tools/testdata/placeholder-errors/input/xliff/es-419.xlf b/packages/localize-tools/testdata/placeholder-errors/input/xliff/es-419.xlf index 5be2a8984a..3c06709a9e 100644 --- a/packages/localize-tools/testdata/placeholder-errors/input/xliff/es-419.xlf +++ b/packages/localize-tools/testdata/placeholder-errors/input/xliff/es-419.xlf @@ -10,10 +10,6 @@ Hello ${name} Hola Mundo - - Hello ${name} - Hola ${alert("evil") || name} - <b>Hello World</b> Hola Mundo From ceb100194aa3b75e771bb8a81c58d92cc795f765 Mon Sep 17 00:00:00 2001 From: Augustine Kim Date: Tue, 30 Jan 2024 17:31:52 -0800 Subject: [PATCH 7/7] Resolve review items - remove extra param from function that replaces expression - keep expression from before normalization for error logging --- packages/localize-tools/src/messages.ts | 43 ++++++++++--------- .../src/tests/e2e/placeholder-errors.test.ts | 4 +- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/packages/localize-tools/src/messages.ts b/packages/localize-tools/src/messages.ts index 9ccfbd534b..8e08fc2d1a 100644 --- a/packages/localize-tools/src/messages.ts +++ b/packages/localize-tools/src/messages.ts @@ -139,26 +139,34 @@ export function validateLocalizedPlaceholders( // Note that two identical placeholders could appear in the same template, // and it matters how many of them there are, hence we use an array, not a // set (might be good to implement some form of multiset here). - const remainingProgramPlaceholders = []; + const remainingProgramPlaceholders: Array<{ + raw: string; + normalized: string; + }> = []; for (const content of programMsg.contents) { if (typeof content !== 'string') { - remainingProgramPlaceholders.push( - replaceExpressionInTemplateString(content.untranslatable) - ); + remainingProgramPlaceholders.push({ + raw: content.untranslatable, + normalized: normalizeExpressionInTemplateString( + content.untranslatable + ), + }); } } for (const content of localizedMsg.contents) { if (typeof content !== 'string') { - const placeholder = replaceExpressionInTemplateString( + const normalizedPlaceholder = normalizeExpressionInTemplateString( content.untranslatable ); - const index = remainingProgramPlaceholders.indexOf(placeholder); + const index = remainingProgramPlaceholders.findIndex( + ({normalized}) => normalized === normalizedPlaceholder + ); if (index === -1) { errors.push( `Placeholder error in ${locale} ` + `localization of ${localizedMsg.name}: ` + - `unexpected "${placeholder}"` + `unexpected "${content.untranslatable}"` ); } else { remainingProgramPlaceholders.splice(index, 1); @@ -170,7 +178,7 @@ export function validateLocalizedPlaceholders( errors.push( `Placeholder error in ${locale} ` + `localization of ${localizedMsg.name}: ` + - `missing "${placeholder}"` + `missing "${placeholder.raw}"` ); } } @@ -179,24 +187,19 @@ export function validateLocalizedPlaceholders( } /** - * Given a template string, replace all expression with a provided string (or - * "expr" if none provided). + * Given a template string, replace all expression with "expr". Used to compare + * static parts of the template string. * * e.g. `hello ${foo} world ${bar}` -> `hello ${expr} world ${expr}` */ -function replaceExpressionInTemplateString( - templateString: string, - expression = 'expr' -): string { +function normalizeExpressionInTemplateString(templateString: string): string { const template = parseStringAsTemplateLiteral(templateString); if (ts.isNoSubstitutionTemplateLiteral(template)) { return template.text; } - const fragments: string[] = []; - fragments.push(template.head.text); - for (let i = 0; i < template.templateSpans.length; i++) { - fragments.push('${' + expression + '}'); - fragments.push(template.templateSpans[i].literal.text); + let normalizedString = template.head.text; + for (const span of template.templateSpans) { + normalizedString += '${expr}' + span.literal.text; } - return fragments.join(''); + return normalizedString; } diff --git a/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts b/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts index 7b5d9b0b8d..88f798559c 100644 --- a/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts +++ b/packages/localize-tools/src/tests/e2e/placeholder-errors.test.ts @@ -12,8 +12,8 @@ e2eGoldensTest( 1, `One or more localized templates contain a set of placeholders (HTML or template literal expressions) that do not exactly match the source code, aborting. Details: -Placeholder error in es-419 localization of extra-expression: unexpected "\${expr}" -Placeholder error in es-419 localization of missing-expression: missing "\${expr}" +Placeholder error in es-419 localization of extra-expression: unexpected "\${alert("evil")}" +Placeholder error in es-419 localization of missing-expression: missing "\${name}" Placeholder error in es-419 localization of missing-html: missing "" Placeholder error in es-419 localization of missing-html: missing "" Placeholder error in es-419 localization of changed-html: unexpected ""