"""Tests to cover the Tools/i18n package"""
import os
import re
import sys
import unittest
from textwrap import dedent
from pathlib import Path
from test.support.script_helper import assert_python_ok
from test.test_tools import skip_if_missing, toolsdir
from test.support.os_helper import temp_cwd, temp_dir
skip_if_missing()
DATA_DIR = Path(__file__).resolve().parent / 'i18n_data'
def normalize_POT_file(pot):
"""Normalize the POT creation timestamp, charset and
file locations to make the POT file easier to compare.
"""
# Normalize the creation date.
date_pattern = re.compile(r'"POT-Creation-Date: .+?\\n"')
header = r'"POT-Creation-Date: 2000-01-01 00:00+0000\\n"'
pot = re.sub(date_pattern, header, pot)
# Normalize charset to UTF-8 (currently there's no way to specify the output charset).
charset_pattern = re.compile(r'"Content-Type: text/plain; charset=.+?\\n"')
charset = r'"Content-Type: text/plain; charset=UTF-8\\n"'
pot = re.sub(charset_pattern, charset, pot)
# Normalize file location path separators in case this test is
# running on Windows (which uses '\').
fileloc_pattern = re.compile(r'#:.+')
def replace(match):
return match[0].replace(os.sep, "/")
pot = re.sub(fileloc_pattern, replace, pot)
return pot
class Test_pygettext(unittest.TestCase):
"""Tests for the pygettext.py tool"""
script = Path(toolsdir, 'i18n', 'pygettext.py')
def get_header(self, data):
""" utility: return the header of a .po file as a dictionary """
headers = {}
for line in data.split('\n'):
if not line or line.startswith(('#', 'msgid', 'msgstr')):
continue
line = line.strip('"')
key, val = line.split(':', 1)
headers[key] = val.strip()
return headers
def get_msgids(self, data):
""" utility: return all msgids in .po file as a list of strings """
msgids = []
reading_msgid = False
cur_msgid = []
for line in data.split('\n'):
if reading_msgid:
if line.startswith('"'):
cur_msgid.append(line.strip('"'))
else:
msgids.append('\n'.join(cur_msgid))
cur_msgid = []
reading_msgid = False
continue
if line.startswith('msgid '):
line = line[len('msgid '):]
cur_msgid.append(line.strip('"'))
reading_msgid = True
else:
if reading_msgid:
msgids.append('\n'.join(cur_msgid))
return msgids
def assert_POT_equal(self, expected, actual):
"""Check if two POT files are equal"""
self.maxDiff = None
self.assertEqual(normalize_POT_file(expected), normalize_POT_file(actual))
def extract_from_str(self, module_content, *, args=(), strict=True):
"""Return all msgids extracted from module_content."""
filename = 'test.py'
with temp_cwd(None):
with open(filename, 'w', encoding='utf-8') as fp:
fp.write(module_content)
res = assert_python_ok('-Xutf8', self.script, *args, filename)
if strict:
self.assertEqual(res.err, b'')
with open('messages.pot', encoding='utf-8') as fp:
data = fp.read()
return self.get_msgids(data)
def extract_docstrings_from_str(self, module_content):
"""Return all docstrings extracted from module_content."""
return self.extract_from_str(module_content, args=('--docstrings',), strict=False)
def test_header(self):
"""Make sure the required fields are in the header, according to:
http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry
"""
with temp_cwd(None) as cwd:
assert_python_ok('-Xutf8', self.script)
with open('messages.pot', encoding='utf-8') as fp:
data = fp.read()
header = self.get_header(data)
self.assertIn("Project-Id-Version", header)
self.assertIn("POT-Creation-Date", header)
self.assertIn("PO-Revision-Date", header)
self.assertIn("Last-Translator", header)
self.assertIn("Language-Team", header)
self.assertIn("MIME-Version", header)
self.assertIn("Content-Type", header)
self.assertIn("Content-Transfer-Encoding", header)
self.assertIn("Generated-By", header)
# not clear if these should be required in POT (template) files
#self.assertIn("Report-Msgid-Bugs-To", header)
#self.assertIn("Language", header)
#"Plural-Forms" is optional
@unittest.skipIf(sys.platform.startswith('aix'),
'bpo-29972: broken test on AIX')
def test_POT_Creation_Date(self):
""" Match the date format from xgettext for POT-Creation-Date """
from datetime import datetime
with temp_cwd(None) as cwd:
assert_python_ok('-Xutf8', self.script)
with open('messages.pot', encoding='utf-8') as fp:
data = fp.read()
header = self.get_header(data)
creationDate = header['POT-Creation-Date']
# peel off the escaped newline at the end of string
if creationDate.endswith('\\n'):
creationDate = creationDate[:-len('\\n')]
# This will raise if the date format does not exactly match.
datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z')
def test_funcdocstring(self):
for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
with self.subTest(doc):
msgids = self.extract_docstrings_from_str(dedent('''\
def foo(bar):
%s
''' % doc))
self.assertIn('doc', msgids)
def test_funcdocstring_bytes(self):
msgids = self.extract_docstrings_from_str(dedent('''\
def foo(bar):
b"""doc"""
'''))
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_funcdocstring_fstring(self):
msgids = self.extract_docstrings_from_str(dedent('''\
def foo(bar):
f"""doc"""
'''))
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_classdocstring(self):
for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
with self.subTest(doc):
msgids = self.extract_docstrings_from_str(dedent('''\
class C:
%s
''' % doc))
self.assertIn('doc', msgids)
def test_classdocstring_bytes(self):
msgids = self.extract_docstrings_from_str(dedent('''\
class C:
b"""doc"""
'''))
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_classdocstring_fstring(self):
msgids = self.extract_docstrings_from_str(dedent('''\
class C:
f"""doc"""
'''))
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_moduledocstring(self):
for doc in ('"""doc"""', "r'''doc'''", "R'doc'", 'u"doc"'):
with self.subTest(doc):
msgids = self.extract_docstrings_from_str(dedent('''\
%s
''' % doc))
self.assertIn('doc', msgids)
def test_moduledocstring_bytes(self):
msgids = self.extract_docstrings_from_str(dedent('''\
b"""doc"""
'''))
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_moduledocstring_fstring(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"""doc"""
'''))
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_msgid(self):
msgids = self.extract_docstrings_from_str(
'''_("""doc""" r'str' u"ing")''')
self.assertIn('docstring', msgids)
def test_msgid_bytes(self):
msgids = self.extract_docstrings_from_str('_(b"""doc""")')
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_msgid_fstring(self):
msgids = self.extract_docstrings_from_str('_(f"""doc""")')
self.assertFalse([msgid for msgid in msgids if 'doc' in msgid])
def test_funcdocstring_annotated_args(self):
""" Test docstrings for functions with annotated args """
msgids = self.extract_docstrings_from_str(dedent('''\
def foo(bar: str):
"""doc"""
'''))
self.assertIn('doc', msgids)
def test_funcdocstring_annotated_return(self):
""" Test docstrings for functions with annotated return type """
msgids = self.extract_docstrings_from_str(dedent('''\
def foo(bar) -> str:
"""doc"""
'''))
self.assertIn('doc', msgids)
def test_funcdocstring_defvalue_args(self):
""" Test docstring for functions with default arg values """
msgids = self.extract_docstrings_from_str(dedent('''\
def foo(bar=()):
"""doc"""
'''))
self.assertIn('doc', msgids)
def test_funcdocstring_multiple_funcs(self):
""" Test docstring extraction for multiple functions combining
annotated args, annotated return types and default arg values
"""
msgids = self.extract_docstrings_from_str(dedent('''\
def foo1(bar: tuple=()) -> str:
"""doc1"""
def foo2(bar: List[1:2]) -> (lambda x: x):
"""doc2"""
def foo3(bar: 'func'=lambda x: x) -> {1: 2}:
"""doc3"""
'''))
self.assertIn('doc1', msgids)
self.assertIn('doc2', msgids)
self.assertIn('doc3', msgids)
def test_classdocstring_early_colon(self):
""" Test docstring extraction for a class with colons occurring within
the parentheses.
"""
msgids = self.extract_docstrings_from_str(dedent('''\
class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)):
"""doc"""
'''))
self.assertIn('doc', msgids)
def test_calls_in_fstrings(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{_('foo bar')}"
'''))
self.assertIn('foo bar', msgids)
def test_calls_in_fstrings_raw(self):
msgids = self.extract_docstrings_from_str(dedent('''\
rf"{_('foo bar')}"
'''))
self.assertIn('foo bar', msgids)
def test_calls_in_fstrings_nested(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"""{f'{_("foo bar")}'}"""
'''))
self.assertIn('foo bar', msgids)
def test_calls_in_fstrings_attribute(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{obj._('foo bar')}"
'''))
self.assertIn('foo bar', msgids)
def test_calls_in_fstrings_with_call_on_call(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{type(str)('foo bar')}"
'''))
self.assertNotIn('foo bar', msgids)
def test_calls_in_fstrings_with_format(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{_('foo {bar}').format(bar='baz')}"
'''))
self.assertIn('foo {bar}', msgids)
def test_calls_in_fstrings_with_wrong_input_1(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{_(f'foo {bar}')}"
'''))
self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid])
def test_calls_in_fstrings_with_wrong_input_2(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{_(1)}"
'''))
self.assertNotIn(1, msgids)
def test_calls_in_fstring_with_multiple_args(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{_('foo', 'bar')}"
'''))
self.assertIn('foo', msgids)
self.assertNotIn('bar', msgids)
def test_calls_in_fstring_with_keyword_args(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{_('foo', bar='baz')}"
'''))
self.assertIn('foo', msgids)
self.assertNotIn('bar', msgids)
self.assertNotIn('baz', msgids)
def test_calls_in_fstring_with_partially_wrong_expression(self):
msgids = self.extract_docstrings_from_str(dedent('''\
f"{_(f'foo') + _('bar')}"
'''))
self.assertNotIn('foo', msgids)
self.assertIn('bar', msgids)
def test_function_and_class_names(self):
"""Test that function and class names are not mistakenly extracted."""
msgids = self.extract_from_str(dedent('''\
def _(x):
pass
def _(x="foo"):
pass
async def _(x):
pass
class _(object):
pass
'''))
self.assertEqual(msgids, [''])
def test_pygettext_output(self):
"""Test that the pygettext output exactly matches snapshots."""
for input_file in DATA_DIR.glob('*.py'):
output_file = input_file.with_suffix('.pot')
with self.subTest(input_file=f'i18n_data/{input_file}'):
contents = input_file.read_text(encoding='utf-8')
with temp_cwd(None):
Path(input_file.name).write_text(contents)
assert_python_ok('-Xutf8', self.script, '--docstrings', input_file.name)
output = Path('messages.pot').read_text(encoding='utf-8')
expected = output_file.read_text(encoding='utf-8')
self.assert_POT_equal(expected, output)
def test_files_list(self):
"""Make sure the directories are inspected for source files
bpo-31920
"""
text1 = 'Text to translate1'
text2 = 'Text to translate2'
text3 = 'Text to ignore'
with temp_cwd(None), temp_dir(None) as sdir:
pymod = Path(sdir, 'pypkg', 'pymod.py')
pymod.parent.mkdir()
pymod.write_text(f'_({text1!r})', encoding='utf-8')
pymod2 = Path(sdir, 'pkg.py', 'pymod2.py')
pymod2.parent.mkdir()
pymod2.write_text(f'_({text2!r})', encoding='utf-8')
pymod3 = Path(sdir, 'CVS', 'pymod3.py')
pymod3.parent.mkdir()
pymod3.write_text(f'_({text3!r})', encoding='utf-8')
assert_python_ok('-Xutf8', self.script, sdir)
data = Path('messages.pot').read_text(encoding='utf-8')
self.assertIn(f'msgid "{text1}"', data)
self.assertIn(f'msgid "{text2}"', data)
self.assertNotIn(text3, data)
def update_POT_snapshots():
for input_file in DATA_DIR.glob('*.py'):
output_file = input_file.with_suffix('.pot')
contents = input_file.read_bytes()
with temp_cwd(None):
Path(input_file.name).write_bytes(contents)
assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings', input_file.name)
output = Path('messages.pot').read_text(encoding='utf-8')
output = normalize_POT_file(output)
output_file.write_text(output, encoding='utf-8')
if __name__ == '__main__':
# To regenerate POT files
if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
update_POT_snapshots()
sys.exit(0)
unittest.main()
Codebase Browser
cpython