Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b6760b7

Browse files
tomasr8tomasr8
authored
gh-130453: pygettext: Allow specifying multiple keywords with the same function name (GH-131380)
1 parent 619edb8 commit b6760b7

5 files changed

Lines changed: 217 additions & 43 deletions

File tree

‎Lib/test/test_tools/i18n_data/multiple_keywords.pot‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# SOME DESCRIPTIVE TITLE.
2+
# Copyright (C) YEAR ORGANIZATION
3+
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
4+
#
5+
msgid ""
6+
msgstr ""
7+
"Project-Id-Version: PACKAGE VERSION\n"
8+
"POT-Creation-Date: 2000-01-01 00:00+0000\n"
9+
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
10+
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
11+
"Language-Team: LANGUAGE <[email protected]>\n"
12+
"MIME-Version: 1.0\n"
13+
"Content-Type: text/plain; charset=UTF-8\n"
14+
"Content-Transfer-Encoding: 8bit\n"
15+
"Generated-By: pygettext.py 1.5\n"
16+
17+
18+
#: multiple_keywords.py:3
19+
msgid "bar"
20+
msgstr ""
21+
22+
#: multiple_keywords.py:5
23+
msgctxt "baz"
24+
msgid "qux"
25+
msgstr ""
26+
27+
#: multiple_keywords.py:9
28+
msgctxt "corge"
29+
msgid "grault"
30+
msgstr ""
31+
32+
#: multiple_keywords.py:11
33+
msgctxt "xyzzy"
34+
msgid "foo"
35+
msgid_plural "foos"
36+
msgstr[0] ""
37+
msgstr[1] ""
38+

‎Lib/test/test_tools/i18n_data/multiple_keywords.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from gettext import gettext as foo
2+
3+
foo('bar')
4+
5+
foo('baz', 'qux')
6+
7+
# The 't' specifier is not supported, so the following
8+
# call is extracted as pgettext instead of ngettext.
9+
foo('corge', 'grault', 1)
10+
11+
foo('xyzzy', 'foo', 'foos', 1)

‎Lib/test/test_tools/test_i18n.py‎

Lines changed: 82 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818

1919

2020
with imports_under_tool("i18n"):
21-
from pygettext import parse_spec
21+
from pygettext import (parse_spec, process_keywords, DEFAULTKEYWORDS,
22+
unparse_spec)
2223

2324

2425
def normalize_POT_file(pot):
@@ -483,20 +484,22 @@ def test_comments_not_extracted_without_tags(self):
483484

484485
def test_parse_keyword_spec(self):
485486
valid = (
486-
('foo', ('foo', {0: 'msgid'})),
487-
('foo:1', ('foo', {0: 'msgid'})),
488-
('foo:1,2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
489-
('foo:1, 2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
490-
('foo:1,2c', ('foo', {0: 'msgid', 1: 'msgctxt'})),
491-
('foo:2c,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
492-
('foo:2c ,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
493-
('foo:1,2,3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
494-
('foo:1, 2, 3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
495-
('foo:3c,1,2', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
487+
('foo', ('foo', {'msgid': 0})),
488+
('foo:1', ('foo', {'msgid': 0})),
489+
('foo:1,2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
490+
('foo:1, 2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
491+
('foo:1,2c', ('foo', {'msgid': 0, 'msgctxt': 1})),
492+
('foo:2c,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
493+
('foo:2c ,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
494+
('foo:1,2,3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
495+
('foo:1, 2, 3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
496+
('foo:3c,1,2', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
496497
)
497498
for spec, expected in valid:
498499
with self.subTest(spec=spec):
499500
self.assertEqual(parse_spec(spec), expected)
501+
# test unparse-parse round-trip
502+
self.assertEqual(parse_spec(unparse_spec(*expected)), expected)
500503

501504
invalid = (
502505
('foo:', "Invalid keyword spec 'foo:': missing argument positions"),
@@ -516,6 +519,70 @@ def test_parse_keyword_spec(self):
516519
parse_spec(spec)
517520
self.assertEqual(str(cm.exception), message)
518521

522+
def test_process_keywords(self):
523+
default_keywords = {name: [spec] for name, spec
524+
in DEFAULTKEYWORDS.items()}
525+
inputs = (
526+
(['foo'], True),
527+
(['_:1,2'], True),
528+
(['foo', 'foo:1,2'], True),
529+
(['foo'], False),
530+
(['_:1,2', '_:1c,2,3', 'pgettext'], False),
531+
# Duplicate entries
532+
(['foo', 'foo'], True),
533+
(['_'], False)
534+
)
535+
expected = (
536+
{'foo': [{'msgid': 0}]},
537+
{'_': [{'msgid': 0, 'msgid_plural': 1}]},
538+
{'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]},
539+
default_keywords | {'foo': [{'msgid': 0}]},
540+
default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1},
541+
{'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2},
542+
{'msgid': 0}],
543+
'pgettext': [{'msgid': 0},
544+
{'msgctxt': 0, 'msgid': 1}]},
545+
{'foo': [{'msgid': 0}]},
546+
default_keywords,
547+
)
548+
for (keywords, no_default_keywords), expected in zip(inputs, expected):
549+
with self.subTest(keywords=keywords,
550+
no_default_keywords=no_default_keywords):
551+
processed = process_keywords(
552+
keywords,
553+
no_default_keywords=no_default_keywords)
554+
self.assertEqual(processed, expected)
555+
556+
def test_multiple_keywords_same_funcname_errors(self):
557+
# If at least one keyword spec for a given funcname matches,
558+
# no error should be printed.
559+
msgids, stderr = self.extract_from_str(dedent('''\
560+
_("foo", 42)
561+
_(42, "bar")
562+
'''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True)
563+
self.assertIn('foo', msgids)
564+
self.assertIn('bar', msgids)
565+
self.assertEqual(stderr, b'')
566+
567+
# If no keyword spec for a given funcname matches,
568+
# all errors are printed.
569+
msgids, stderr = self.extract_from_str(dedent('''\
570+
_(x, 42)
571+
_(42, y)
572+
'''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True,
573+
strict=False)
574+
self.assertEqual(msgids, [''])
575+
# Normalize line endings on Windows
576+
stderr = stderr.decode('utf-8').replace('\r', '')
577+
self.assertEqual(
578+
stderr,
579+
'*** test.py:1: No keywords matched gettext call "_":\n'
580+
'\tkeyword="_": Expected a string constant for argument 1, got x\n'
581+
'\tkeyword="_:2": Expected a string constant for argument 2, got 42\n'
582+
'*** test.py:2: No keywords matched gettext call "_":\n'
583+
'\tkeyword="_": Expected a string constant for argument 1, got 42\n'
584+
'\tkeyword="_:2": Expected a string constant for argument 2, got y\n')
585+
519586

520587
def extract_from_snapshots():
521588
snapshots = {
@@ -526,6 +593,10 @@ def extract_from_snapshots():
526593
'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2',
527594
'--keyword=pfoo:1c,2',
528595
'--keyword=npfoo:1c,2,3', '--keyword=_:1,2'),
596+
'multiple_keywords.py': ('--keyword=foo:1c,2,3', '--keyword=foo:1c,2',
597+
'--keyword=foo:1,2',
598+
# repeat a keyword to make sure it is extracted only once
599+
'--keyword=foo', '--keyword=foo'),
529600
# == Test character escaping
530601
# Escape ascii and unicode:
531602
'escapes.py': ('--escape', '--add-comments='),

‎Misc/NEWS.d/next/Tools-Demos/2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Allow passing multiple keyword arguments with the same function name in
2+
:program:`pygettext`.

‎Tools/i18n/pygettext.py‎

Lines changed: 84 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -282,15 +282,15 @@ def getFilesForName(name):
282282
# Key is the function name, value is a dictionary mapping argument positions to the
283283
# type of the argument. The type is one of 'msgid', 'msgid_plural', or 'msgctxt'.
284284
DEFAULTKEYWORDS = {
285-
'_': {0: 'msgid'},
286-
'gettext': {0: 'msgid'},
287-
'ngettext': {0: 'msgid', 1: 'msgid_plural'},
288-
'pgettext': {0: 'msgctxt', 1: 'msgid'},
289-
'npgettext': {0: 'msgctxt', 1: 'msgid', 2: 'msgid_plural'},
290-
'dgettext': {1: 'msgid'},
291-
'dngettext': {1: 'msgid', 2: 'msgid_plural'},
292-
'dpgettext': {1: 'msgctxt', 2: 'msgid'},
293-
'dnpgettext': {1: 'msgctxt', 2: 'msgid', 3: 'msgid_plural'},
285+
'_': {'msgid': 0},
286+
'gettext': {'msgid': 0},
287+
'ngettext': {'msgid': 0, 'msgid_plural': 1},
288+
'pgettext': {'msgctxt': 0, 'msgid': 1},
289+
'npgettext': {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2},
290+
'dgettext': {'msgid': 1},
291+
'dngettext': {'msgid': 1, 'msgid_plural': 2},
292+
'dpgettext': {'msgctxt': 1, 'msgid': 2},
293+
'dnpgettext': {'msgctxt': 1, 'msgid': 2, 'msgid_plural': 3},
294294
}
295295

296296

@@ -327,7 +327,7 @@ def parse_spec(spec):
327327
parts = spec.strip().split(':', 1)
328328
if len(parts) == 1:
329329
name = parts[0]
330-
return name, {0: 'msgid'}
330+
return name, {'msgid': 0}
331331

332332
name, args = parts
333333
if not args:
@@ -373,7 +373,41 @@ def parse_spec(spec):
373373
raise ValueError(f'Invalid keyword spec {spec!r}: '
374374
'msgctxt cannot appear without msgid')
375375

376-
return name, {v: k for k, v in result.items()}
376+
return name, result
377+
378+
379+
def unparse_spec(name, spec):
380+
"""Unparse a keyword spec dictionary into a string."""
381+
if spec == {'msgid': 0}:
382+
return name
383+
384+
parts = []
385+
for arg, pos in sorted(spec.items(), key=lambda x: x[1]):
386+
if arg == 'msgctxt':
387+
parts.append(f'{pos + 1}c')
388+
else:
389+
parts.append(str(pos + 1))
390+
return f'{name}:{','.join(parts)}'
391+
392+
393+
def process_keywords(keywords, *, no_default_keywords):
394+
custom_keywords = {}
395+
for spec in dict.fromkeys(keywords):
396+
name, spec = parse_spec(spec)
397+
if name not in custom_keywords:
398+
custom_keywords[name] = []
399+
custom_keywords[name].append(spec)
400+
401+
if no_default_keywords:
402+
return custom_keywords
403+
404+
# custom keywords override default keywords
405+
for name, spec in DEFAULTKEYWORDS.items():
406+
if name not in custom_keywords:
407+
custom_keywords[name] = []
408+
if spec not in custom_keywords[name]:
409+
custom_keywords[name].append(spec)
410+
return custom_keywords
377411

378412

379413
@dataclass(frozen=True)
@@ -459,32 +493,53 @@ def _extract_docstring(self, node):
459493

460494
def _extract_message(self, node):
461495
func_name = self._get_func_name(node)
462-
spec = self.options.keywords.get(func_name)
463-
if spec is None:
496+
errors = []
497+
specs = self.options.keywords.get(func_name, [])
498+
for spec in specs:
499+
err = self._extract_message_with_spec(node, spec)
500+
if err is None:
501+
return
502+
errors.append(err)
503+
504+
if not errors:
464505
return
506+
if len(errors) == 1:
507+
print(f'*** {self.filename}:{node.lineno}: {errors[0]}',
508+
file=sys.stderr)
509+
else:
510+
# There are multiple keyword specs for the function name and
511+
# none of them could be extracted. Print a general error
512+
# message and list the errors for each keyword spec.
513+
print(f'*** {self.filename}:{node.lineno}: '
514+
f'No keywords matched gettext call "{func_name}":',
515+
file=sys.stderr)
516+
for spec, err in zip(specs, errors, strict=True):
517+
unparsed = unparse_spec(func_name, spec)
518+
print(f'\tkeyword="{unparsed}": {err}', file=sys.stderr)
519+
520+
def _extract_message_with_spec(self, node, spec):
521+
"""Extract a gettext call with the given spec.
465522
466-
max_index = max(spec)
523+
Return None if the gettext call was successfully extracted,
524+
otherwise return an error message.
525+
"""
526+
max_index = max(spec.values())
467527
has_var_positional = any(isinstance(arg, ast.Starred) for
468528
arg in node.args[:max_index+1])
469529
if has_var_positional:
470-
print(f'*** {self.filename}:{node.lineno}: Variable positional '
471-
f'arguments are not allowed in gettext calls', file=sys.stderr)
472-
return
530+
return ('Variable positional arguments are not '
531+
'allowed in gettext calls')
473532

474533
if max_index >= len(node.args):
475-
print(f'*** {self.filename}:{node.lineno}: Expected at least '
476-
f'{max(spec) + 1} positional argument(s) in gettext call, '
477-
f'got {len(node.args)}', file=sys.stderr)
478-
return
534+
return (f'Expected at least {max_index + 1} positional '
535+
f'argument(s) in gettext call, got {len(node.args)}')
479536

480537
msg_data = {}
481-
for position, arg_type in spec.items():
538+
for arg_type, position in spec.items():
482539
arg = node.args[position]
483540
if not self._is_string_const(arg):
484-
print(f'*** {self.filename}:{arg.lineno}: Expected a string '
485-
f'constant for argument {position + 1}, '
486-
f'got {ast.unparse(arg)}', file=sys.stderr)
487-
return
541+
return (f'Expected a string constant for argument '
542+
f'{position + 1}, got {ast.unparse(arg)}')
488543
msg_data[arg_type] = arg.value
489544

490545
lineno = node.lineno
@@ -729,15 +784,12 @@ class Options:
729784

730785
# calculate all keywords
731786
try:
732-
custom_keywords = dict(parse_spec(spec) for spec in options.keywords)
787+
options.keywords = process_keywords(
788+
options.keywords,
789+
no_default_keywords=no_default_keywords)
733790
except ValueError as e:
734791
print(e, file=sys.stderr)
735792
sys.exit(1)
736-
options.keywords = {}
737-
if not no_default_keywords:
738-
options.keywords |= DEFAULTKEYWORDS
739-
# custom keywords override default keywords
740-
options.keywords |= custom_keywords
741793

742794
# initialize list of strings to exclude
743795
if options.excludefilename:

0 commit comments

Comments
 (0)