From 7e4bbb55804c8bebedd410c6d9c49d9414c6ad43 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Mon, 17 Feb 2025 22:49:48 +0100 Subject: [PATCH 1/7] Add human-readable snapshots --- Lib/test/test_tools/msgfmt_data/fuzzy.json | 1 + Lib/test/test_tools/msgfmt_data/general.json | 58 ++++++++++++++++++++ Lib/test/test_tools/test_msgfmt.py | 10 ++++ 3 files changed, 69 insertions(+) create mode 100644 Lib/test/test_tools/msgfmt_data/fuzzy.json create mode 100644 Lib/test/test_tools/msgfmt_data/general.json diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.json b/Lib/test/test_tools/msgfmt_data/fuzzy.json new file mode 100644 index 00000000000000..fe51488c7066f6 --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/fuzzy.json @@ -0,0 +1 @@ +[] diff --git a/Lib/test/test_tools/msgfmt_data/general.json b/Lib/test/test_tools/msgfmt_data/general.json new file mode 100644 index 00000000000000..64d6d70d82923f --- /dev/null +++ b/Lib/test/test_tools/msgfmt_data/general.json @@ -0,0 +1,58 @@ +[ + [ + "", + "Project-Id-Version: PACKAGE VERSION\nPOT-Creation-Date: 2024-10-26 18:06+0200\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME \nLanguage-Team: LANGUAGE \nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n" + ], + [ + [ + "abc\u0004One email sent.", + 0 + ], + "One email sent." + ], + [ + [ + "abc\u0004One email sent.", + 1 + ], + "%d emails sent." + ], + [ + "abc\u0004foo", + "bar" + ], + [ + "bar", + "baz" + ], + [ + "xyz\u0004\n newlines \n", + "\n translated \n" + ], + [ + "xyz\u0004\"escapes\"", + "\"translated\"" + ], + [ + "xyz\u0004Multilinestring", + "Multilinetranslation" + ], + [ + [ + "xyz\u0004One email sent.", + 0 + ], + "One email sent." + ], + [ + [ + "xyz\u0004One email sent.", + 1 + ], + "%d emails sent." + ], + [ + "xyz\u0004foo", + "bar" + ] +] diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py index e3e3035c4f4395..ea3d2bc2ebfb32 100644 --- a/Lib/test/test_tools/test_msgfmt.py +++ b/Lib/test/test_tools/test_msgfmt.py @@ -1,5 +1,6 @@ """Tests for the Tools/i18n/msgfmt.py tool.""" +import json import sys import unittest from gettext import GNUTranslations @@ -117,6 +118,15 @@ def update_catalog_snapshots(): for po_file in data_dir.glob('*.po'): mo_file = po_file.with_suffix('.mo') compile_messages(po_file, mo_file) + # Create a human-readable JSON file which is + # easier to review than the binary .mo file. + with open(mo_file, 'rb') as f: + translations = GNUTranslations(f) + catalog_file = po_file.with_suffix('.json') + with open(catalog_file, 'w') as f: + data = list(translations._catalog.items()) + json.dump(data, f, indent=4) + f.write('\n') if __name__ == '__main__': From 69402a5c23eb1b775bc417b3ff6b9442a206d534 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Mon, 17 Feb 2025 22:50:23 +0100 Subject: [PATCH 2/7] Reset context when adding a message --- Lib/test/test_tools/msgfmt_data/general.json | 48 +++++++++---------- Lib/test/test_tools/msgfmt_data/general.mo | Bin 728 -> 712 bytes Tools/i18n/msgfmt.py | 1 + 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/Lib/test/test_tools/msgfmt_data/general.json b/Lib/test/test_tools/msgfmt_data/general.json index 64d6d70d82923f..803efec7a3a302 100644 --- a/Lib/test/test_tools/msgfmt_data/general.json +++ b/Lib/test/test_tools/msgfmt_data/general.json @@ -3,54 +3,54 @@ "", "Project-Id-Version: PACKAGE VERSION\nPOT-Creation-Date: 2024-10-26 18:06+0200\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME \nLanguage-Team: LANGUAGE \nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n" ], + [ + "\n newlines \n", + "\n translated \n" + ], + [ + "\"escapes\"", + "\"translated\"" + ], + [ + "Multilinestring", + "Multilinetranslation" + ], [ [ - "abc\u0004One email sent.", + "One email sent.", 0 ], "One email sent." ], [ [ - "abc\u0004One email sent.", + "One email sent.", 1 ], "%d emails sent." ], - [ - "abc\u0004foo", - "bar" - ], - [ - "bar", - "baz" - ], - [ - "xyz\u0004\n newlines \n", - "\n translated \n" - ], - [ - "xyz\u0004\"escapes\"", - "\"translated\"" - ], - [ - "xyz\u0004Multilinestring", - "Multilinetranslation" - ], [ [ - "xyz\u0004One email sent.", + "abc\u0004One email sent.", 0 ], "One email sent." ], [ [ - "xyz\u0004One email sent.", + "abc\u0004One email sent.", 1 ], "%d emails sent." ], + [ + "abc\u0004foo", + "bar" + ], + [ + "bar", + "baz" + ], [ "xyz\u0004foo", "bar" diff --git a/Lib/test/test_tools/msgfmt_data/general.mo b/Lib/test/test_tools/msgfmt_data/general.mo index bc0683a62d0ddaecbc751b9a59eb1fe19bb7619c..44b7363071a98bfe61eadc6adb0b434f1fd36125 100644 GIT binary patch delta 215 zcmcb?dV+O=N<9w)1H)P%<^wU}Rum2GR;Z8pKxt z(sDqo1H^nlzBdyC1IWB6AT0u<(}6U|oEj)S2S|hT?}5?>m>3Mf0!M%X3|tC%spUDD zd8x$;TntL7#mR{Usl`eRzNI-OnPB0PqRhN>hKUO1&}z1H)P%W(VSnK+FupH-T6Hi0=Y1FA%>2VqqZu1H>Tp0*ny#CO}#a zh;4va9w;8o1kqm&rR#w-A5gp*Nb>;cNkCczNY4k-AoW|AfYKlfuK)!Y5|ffyCf6`2 zOqO62Fy>OoOD)gI%u6j+;9{t#tYT40Ely4>NG(0mLC-pwkE U>Wq`;FlkKoW(s6UN-Sak05L!){{R30 diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index 3f731e941eafe7..c0da976979848a 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -149,6 +149,7 @@ def make(filename, outfile): elif l.startswith('msgid') and not l.startswith('msgid_plural'): if section == STR: add(msgctxt, msgid, msgstr, fuzzy) + msgctxt = None if not msgid: # See whether there is an encoding declaration p = HeaderParser() From 741c35c56350d4255dfc33149eb5011e5642cdbd Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Mon, 24 Feb 2025 21:36:59 +0100 Subject: [PATCH 3/7] Add news entry --- .../Tools-Demos/2025-02-24-21-36-23.gh-issue-85012.9K1U0E.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-02-24-21-36-23.gh-issue-85012.9K1U0E.rst diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-02-24-21-36-23.gh-issue-85012.9K1U0E.rst b/Misc/NEWS.d/next/Tools-Demos/2025-02-24-21-36-23.gh-issue-85012.9K1U0E.rst new file mode 100644 index 00000000000000..5ec20583527367 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-02-24-21-36-23.gh-issue-85012.9K1U0E.rst @@ -0,0 +1 @@ +Correctly reset ``msgctxt`` when compiling messages in :program:`msgfmt`. From bcb85337770b8a734ff90c3f02398f71e19f3b8e Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Tue, 25 Feb 2025 22:59:04 +0100 Subject: [PATCH 4/7] Use nicer sorting for snapshots --- Lib/test/test_tools/test_msgfmt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py index ea3d2bc2ebfb32..72e3060de9ea94 100644 --- a/Lib/test/test_tools/test_msgfmt.py +++ b/Lib/test/test_tools/test_msgfmt.py @@ -124,7 +124,8 @@ def update_catalog_snapshots(): translations = GNUTranslations(f) catalog_file = po_file.with_suffix('.json') with open(catalog_file, 'w') as f: - data = list(translations._catalog.items()) + data = translations._catalog.items() + data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0])) json.dump(data, f, indent=4) f.write('\n') From bb685383732607bbb7886e67ded6ec5989ce1ba7 Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Tue, 25 Feb 2025 22:59:50 +0100 Subject: [PATCH 5/7] Temporarily revert fix and regenerate snapshots --- Lib/test/test_tools/msgfmt_data/general.json | 38 +++++++++---------- Lib/test/test_tools/msgfmt_data/general.mo | Bin 712 -> 728 bytes Tools/i18n/msgfmt.py | 1 - 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_tools/msgfmt_data/general.json b/Lib/test/test_tools/msgfmt_data/general.json index 803efec7a3a302..e350b0d4a82e87 100644 --- a/Lib/test/test_tools/msgfmt_data/general.json +++ b/Lib/test/test_tools/msgfmt_data/general.json @@ -4,55 +4,55 @@ "Project-Id-Version: PACKAGE VERSION\nPOT-Creation-Date: 2024-10-26 18:06+0200\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME \nLanguage-Team: LANGUAGE \nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n" ], [ - "\n newlines \n", + "abc\u0004foo", + "bar" + ], + [ + "bar", + "baz" + ], + [ + "xyz\u0004\n newlines \n", "\n translated \n" ], [ - "\"escapes\"", + "xyz\u0004\"escapes\"", "\"translated\"" ], [ - "Multilinestring", + "xyz\u0004Multilinestring", "Multilinetranslation" ], + [ + "xyz\u0004foo", + "bar" + ], [ [ - "One email sent.", + "abc\u0004One email sent.", 0 ], "One email sent." ], [ [ - "One email sent.", + "abc\u0004One email sent.", 1 ], "%d emails sent." ], [ [ - "abc\u0004One email sent.", + "xyz\u0004One email sent.", 0 ], "One email sent." ], [ [ - "abc\u0004One email sent.", + "xyz\u0004One email sent.", 1 ], "%d emails sent." - ], - [ - "abc\u0004foo", - "bar" - ], - [ - "bar", - "baz" - ], - [ - "xyz\u0004foo", - "bar" ] ] diff --git a/Lib/test/test_tools/msgfmt_data/general.mo b/Lib/test/test_tools/msgfmt_data/general.mo index 44b7363071a98bfe61eadc6adb0b434f1fd36125..bc0683a62d0ddaecbc751b9a59eb1fe19bb7619c 100644 GIT binary patch delta 231 zcmX@XdV_U>O1&}z1H)P%W(VSnK+FupH-T6Hi0=Y1FA%>2VqqZu1H>Tp0*ny#CO}#a zh;4va9w;8o1kqm&rR#w-A5gp*Nb>;cNkCczNY4k-AoW|AfYKlfuK)!Y5|ffyCf6`2 zOqO62Fy>OoOD)gI%u6j+;9{t#tYT40Ely4>NG(0mLC-pwkE U>Wq`;FlkKoW(s6UN-Sak05L!){{R30 delta 215 zcmcb?dV+O=N<9w)1H)P%<^wU}Rum2GR;Z8pKxt z(sDqo1H^nlzBdyC1IWB6AT0u<(}6U|oEj)S2S|hT?}5?>m>3Mf0!M%X3|tC%spUDD zd8x$;TntL7#mR{Usl`eRzNI-OnPB0PqRhN>hKU Date: Tue, 25 Feb 2025 23:01:27 +0100 Subject: [PATCH 6/7] Re-apply fix and regenerate snapshots --- Lib/test/test_tools/msgfmt_data/general.json | 28 +++++++++---------- Lib/test/test_tools/msgfmt_data/general.mo | Bin 728 -> 712 bytes Tools/i18n/msgfmt.py | 1 + 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_tools/msgfmt_data/general.json b/Lib/test/test_tools/msgfmt_data/general.json index e350b0d4a82e87..8ceb34cd17fb07 100644 --- a/Lib/test/test_tools/msgfmt_data/general.json +++ b/Lib/test/test_tools/msgfmt_data/general.json @@ -4,24 +4,24 @@ "Project-Id-Version: PACKAGE VERSION\nPOT-Creation-Date: 2024-10-26 18:06+0200\nPO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\nLast-Translator: FULL NAME \nLanguage-Team: LANGUAGE \nMIME-Version: 1.0\nContent-Type: text/plain; charset=UTF-8\nContent-Transfer-Encoding: 8bit\n" ], [ - "abc\u0004foo", - "bar" + "\n newlines \n", + "\n translated \n" ], [ - "bar", - "baz" + "\"escapes\"", + "\"translated\"" ], [ - "xyz\u0004\n newlines \n", - "\n translated \n" + "Multilinestring", + "Multilinetranslation" ], [ - "xyz\u0004\"escapes\"", - "\"translated\"" + "abc\u0004foo", + "bar" ], [ - "xyz\u0004Multilinestring", - "Multilinetranslation" + "bar", + "baz" ], [ "xyz\u0004foo", @@ -29,28 +29,28 @@ ], [ [ - "abc\u0004One email sent.", + "One email sent.", 0 ], "One email sent." ], [ [ - "abc\u0004One email sent.", + "One email sent.", 1 ], "%d emails sent." ], [ [ - "xyz\u0004One email sent.", + "abc\u0004One email sent.", 0 ], "One email sent." ], [ [ - "xyz\u0004One email sent.", + "abc\u0004One email sent.", 1 ], "%d emails sent." diff --git a/Lib/test/test_tools/msgfmt_data/general.mo b/Lib/test/test_tools/msgfmt_data/general.mo index bc0683a62d0ddaecbc751b9a59eb1fe19bb7619c..44b7363071a98bfe61eadc6adb0b434f1fd36125 100644 GIT binary patch delta 215 zcmcb?dV+O=N<9w)1H)P%<^wU}Rum2GR;Z8pKxt z(sDqo1H^nlzBdyC1IWB6AT0u<(}6U|oEj)S2S|hT?}5?>m>3Mf0!M%X3|tC%spUDD zd8x$;TntL7#mR{Usl`eRzNI-OnPB0PqRhN>hKUO1&}z1H)P%W(VSnK+FupH-T6Hi0=Y1FA%>2VqqZu1H>Tp0*ny#CO}#a zh;4va9w;8o1kqm&rR#w-A5gp*Nb>;cNkCczNY4k-AoW|AfYKlfuK)!Y5|ffyCf6`2 zOqO62Fy>OoOD)gI%u6j+;9{t#tYT40Ely4>NG(0mLC-pwkE U>Wq`;FlkKoW(s6UN-Sak05L!){{R30 diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index 3f731e941eafe7..c0da976979848a 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -149,6 +149,7 @@ def make(filename, outfile): elif l.startswith('msgid') and not l.startswith('msgid_plural'): if section == STR: add(msgctxt, msgid, msgstr, fuzzy) + msgctxt = None if not msgid: # See whether there is an encoding declaration p = HeaderParser() From 97d0a0fe05579fa8a8e8bdd58a5ea8d5772cef1f Mon Sep 17 00:00:00 2001 From: Tomas Roun Date: Tue, 25 Feb 2025 23:03:36 +0100 Subject: [PATCH 7/7] Add regression tests --- Lib/test/test_tools/test_msgfmt.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_tools/test_msgfmt.py b/Lib/test/test_tools/test_msgfmt.py index 72e3060de9ea94..8cd31680f76424 100644 --- a/Lib/test/test_tools/test_msgfmt.py +++ b/Lib/test/test_tools/test_msgfmt.py @@ -40,6 +40,28 @@ def test_compilation(self): self.assertDictEqual(actual._catalog, expected._catalog) + def test_translations(self): + with open(data_dir / 'general.mo', 'rb') as f: + t = GNUTranslations(f) + + self.assertEqual(t.gettext('foo'), 'foo') + self.assertEqual(t.gettext('bar'), 'baz') + self.assertEqual(t.pgettext('abc', 'foo'), 'bar') + self.assertEqual(t.pgettext('xyz', 'foo'), 'bar') + self.assertEqual(t.gettext('Multilinestring'), 'Multilinetranslation') + self.assertEqual(t.gettext('"escapes"'), '"translated"') + self.assertEqual(t.gettext('\n newlines \n'), '\n translated \n') + self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 1), + 'One email sent.') + self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 2), + '%d emails sent.') + self.assertEqual(t.npgettext('abc', 'One email sent.', + '%d emails sent.', 1), + 'One email sent.') + self.assertEqual(t.npgettext('abc', 'One email sent.', + '%d emails sent.', 2), + '%d emails sent.') + def test_invalid_msgid_plural(self): with temp_cwd(): Path('invalid.po').write_text('''\