gettext.py

"""Internationalization and localization support.

This module provides internationalization (I18N) and localization (L10N)
support for your Python programs by providing an interface to the GNU gettext
message catalog library.

I18N refers to the operation by which a program is made aware of multiple
languages. L10N refers to the adaptation of your program, once
internationalized, to the local language and cultural habits.

"""

# This module represents the integration of work, contributions, feedback, and
# suggestions from the following people:
#
# Martin von Loewis, who wrote the initial implementation of the underlying
# C-based libintlmodule (later renamed _gettext), along with a skeletal
# gettext.py implementation.
#
# Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
# which also included a pure-Python implementation to read .mo files if
# intlmodule wasn't available.
#
# James Henstridge, who also wrote a gettext.py module, which has some
# interesting, but currently unsupported experimental features: the notion of
# a Catalog class and instances, and the ability to add to a catalog file via
# a Python API.
#
# Barry Warsaw integrated these modules, wrote the .install() API and code,
# and conformed all C and Python code to Python's coding standards.
#
# Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
# module.
#
# J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
#
# TODO:
# - Lazy loading of .mo files. Currently the entire catalog is loaded into
# memory, but that's probably bad for large translated programs. Instead,
# the lexical sort of original strings in GNU .mo files should be exploited
# to do binary searches and lazy initializations. Or you might want to use
# the undocumented double-hash algorithm for .mo files with hash tables, but
# you'll need to study the GNU gettext code to do this.
#
# - Support Solaris .mo file formats. Unfortunately, we've been unable to
# find this format documented anywhere.


importlocale
importos
importre
importsys


__all__= ['NullTranslations', 'GNUTranslations', 'Catalog',
'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
'bind_textdomain_codeset',
'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
'ldngettext', 'lngettext', 'ngettext',
 ]

_default_localedir=os.path.join(sys.base_prefix, 'share', 'locale')

# Expression parsing for plural form selection.
#
# The gettext library supports a small subset of C syntax. The only
# incompatible difference is that integer literals starting with zero are
# decimal.
#
# https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms
# http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y

_token_pattern=re.compile(r"""
 (?P<WHITESPACES>[ \t]+) | # spaces and horizontal tabs
 (?P<NUMBER>[0-9]+\b) | # decimal integer
 (?P<NAME>n\b) | # only n is allowed
 (?P<PARENTHESIS>[()]) |
 (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >,
 # <=, >=, ==, !=, &&, ||,
 # ? :
 # unary and bitwise ops
 # not allowed
 (?P<INVALID>\w+|.) # invalid token
 """, re.VERBOSE|re.DOTALL)

def_tokenize(plural):
formoinre.finditer(_token_pattern, plural):
kind=mo.lastgroup
ifkind=='WHITESPACES':
continue
value=mo.group(kind)
ifkind=='INVALID':
raiseValueError('invalid token in plural form: %s'%value)
yieldvalue
yield''

def_error(value):
ifvalue:
returnValueError('unexpected token in plural form: %s'%value)
else:
returnValueError('unexpected end of plural form')

_binary_ops= (
 ('||',),
 ('&&',),
 ('==', '!='),
 ('<', '>', '<=', '>='),
 ('+', '-'),
 ('*', '/', '%'),
)
_binary_ops= {op: ifori, opsinenumerate(_binary_ops, 1) foropinops}
_c2py_ops= {'||': 'or', '&&': 'and', '/': '//'}

def_parse(tokens, priority=-1):
result=''
nexttok=next(tokens)
whilenexttok=='!':
result+='not '
nexttok=next(tokens)

ifnexttok=='(':
sub, nexttok=_parse(tokens)
result='%s(%s)'% (result, sub)
ifnexttok!=')':
raiseValueError('unbalanced parenthesis in plural form')
elifnexttok=='n':
result='%s%s'% (result, nexttok)
else:
try:
value=int(nexttok, 10)
exceptValueError:
raise_error(nexttok) fromNone
result='%s%d'% (result, value)
nexttok=next(tokens)

j=100
whilenexttokin_binary_ops:
i=_binary_ops[nexttok]
ifi<priority:
break
# Break chained comparisons
ifiin (3, 4) andjin (3, 4): # '==', '!=', '<', '>', '<=', '>='
result='(%s)'%result
# Replace some C operators by their Python equivalents
op=_c2py_ops.get(nexttok, nexttok)
right, nexttok=_parse(tokens, i+1)
result='%s %s %s'% (result, op, right)
j=i
ifj==priority==4: # '<', '>', '<=', '>='
result='(%s)'%result

ifnexttok=='?'andpriority<=0:
if_true, nexttok=_parse(tokens, 0)
ifnexttok!=':':
raise_error(nexttok)
if_false, nexttok=_parse(tokens)
result='%s if %s else %s'% (if_true, result, if_false)
ifpriority==0:
result='(%s)'%result

returnresult, nexttok

def_as_int(n):
try:
i=round(n)
exceptTypeError:
raiseTypeError('Plural value must be an integer, got %s'%
 (n.__class__.__name__,)) fromNone
importwarnings
warnings.warn('Plural value must be an integer, got %s'%
 (n.__class__.__name__,),
DeprecationWarning, 4)
returnn

defc2py(plural):
"""Gets a C expression as used in PO files for plural forms and returns a
 Python function that implements an equivalent expression.
 """

iflen(plural) >1000:
raiseValueError('plural form expression is too long')
try:
result, nexttok=_parse(_tokenize(plural))
ifnexttok:
raise_error(nexttok)

depth=0
forcinresult:
ifc=='(':
depth+=1
ifdepth>20:
# Python compiler limit is about 90.
# The most complex example has 2.
raiseValueError('plural form expression is too complex')
elifc==')':
depth-=1

ns= {'_as_int': _as_int}
exec('''if True:
 def func(n):
 if not isinstance(n, int):
 n = _as_int(n)
 return int(%s)
 '''%result, ns)
returnns['func']
exceptRecursionError:
# Recursion error can be raised in _parse() or exec().
raiseValueError('plural form expression is too complex')


def_expand_lang(loc):
loc=locale.normalize(loc)
COMPONENT_CODESET=1<<0
COMPONENT_TERRITORY=1<<1
COMPONENT_MODIFIER=1<<2
# split up the locale into its base components
mask=0
pos=loc.find('@')
ifpos>=0:
modifier=loc[pos:]
loc=loc[:pos]
mask|=COMPONENT_MODIFIER
else:
modifier=''
pos=loc.find('.')
ifpos>=0:
codeset=loc[pos:]
loc=loc[:pos]
mask|=COMPONENT_CODESET
else:
codeset=''
pos=loc.find('_')
ifpos>=0:
territory=loc[pos:]
loc=loc[:pos]
mask|=COMPONENT_TERRITORY
else:
territory=''
language=loc
ret= []
foriinrange(mask+1):
ifnot (i&~mask): # if all components for this combo exist ...
val=language
ifi&COMPONENT_TERRITORY: val+=territory
ifi&COMPONENT_CODESET: val+=codeset
ifi&COMPONENT_MODIFIER: val+=modifier
ret.append(val)
ret.reverse()
returnret



classNullTranslations:
def__init__(self, fp=None):
self._info= {}
self._charset=None
self._output_charset=None
self._fallback=None
iffpisnotNone:
self._parse(fp)

def_parse(self, fp):
pass

defadd_fallback(self, fallback):
ifself._fallback:
self._fallback.add_fallback(fallback)
else:
self._fallback=fallback

defgettext(self, message):
ifself._fallback:
returnself._fallback.gettext(message)
returnmessage

deflgettext(self, message):
ifself._fallback:
returnself._fallback.lgettext(message)
ifself._output_charset:
returnmessage.encode(self._output_charset)
returnmessage.encode(locale.getpreferredencoding())

defngettext(self, msgid1, msgid2, n):
ifself._fallback:
returnself._fallback.ngettext(msgid1, msgid2, n)
ifn==1:
returnmsgid1
else:
returnmsgid2

deflngettext(self, msgid1, msgid2, n):
ifself._fallback:
returnself._fallback.lngettext(msgid1, msgid2, n)
ifn==1:
tmsg=msgid1
else:
tmsg=msgid2
ifself._output_charset:
returntmsg.encode(self._output_charset)
returntmsg.encode(locale.getpreferredencoding())

definfo(self):
returnself._info

defcharset(self):
returnself._charset

defoutput_charset(self):
returnself._output_charset

defset_output_charset(self, charset):
self._output_charset=charset

definstall(self, names=None):
importbuiltins
builtins.__dict__['_'] =self.gettext
ifhasattr(names, "__contains__"):
if"gettext"innames:
builtins.__dict__['gettext'] =builtins.__dict__['_']
if"ngettext"innames:
builtins.__dict__['ngettext'] =self.ngettext
if"lgettext"innames:
builtins.__dict__['lgettext'] =self.lgettext
if"lngettext"innames:
builtins.__dict__['lngettext'] =self.lngettext


classGNUTranslations(NullTranslations):
# Magic number of .mo files
LE_MAGIC=0x950412de
BE_MAGIC=0xde120495

# Acceptable .mo versions
VERSIONS= (0, 1)

def_get_versions(self, version):
"""Returns a tuple of major version, minor version"""
return (version>>16, version&0xffff)

def_parse(self, fp):
"""Override this method to support alternative .mo formats."""
# Delay struct import for speeding up gettext import when .mo files
# are not used.
fromstructimportunpack
filename=getattr(fp, 'name', '')
# Parse the .mo file header, which consists of 5 little endian 32
# bit words.
self._catalog=catalog= {}
self.plural=lambdan: int(n!=1) # germanic plural by default
buf=fp.read()
buflen=len(buf)
# Are we big endian or little endian?
magic=unpack('<I', buf[:4])[0]
ifmagic==self.LE_MAGIC:
version, msgcount, masteridx, transidx=unpack('<4I', buf[4:20])
ii='<II'
elifmagic==self.BE_MAGIC:
version, msgcount, masteridx, transidx=unpack('>4I', buf[4:20])
ii='>II'
else:
raiseOSError(0, 'Bad magic number', filename)

major_version, minor_version=self._get_versions(version)

ifmajor_versionnotinself.VERSIONS:
raiseOSError(0, 'Bad version number '+str(major_version), filename)

# Now put all messages from the .mo file buffer into the catalog
# dictionary.
foriinrange(0, msgcount):
mlen, moff=unpack(ii, buf[masteridx:masteridx+8])
mend=moff+mlen
tlen, toff=unpack(ii, buf[transidx:transidx+8])
tend=toff+tlen
ifmend<buflenandtend<buflen:
msg=buf[moff:mend]
tmsg=buf[toff:tend]
else:
raiseOSError(0, 'File is corrupt', filename)
# See if we're looking at GNU .mo conventions for metadata
ifmlen==0:
# Catalog description
lastk=None
forb_itemintmsg.split(b'\n'):
item=b_item.decode().strip()
ifnotitem:
continue
k=v=None
if':'initem:
k, v=item.split(':', 1)
k=k.strip().lower()
v=v.strip()
self._info[k] =v
lastk=k
eliflastk:
self._info[lastk] +='\n'+item
ifk=='content-type':
self._charset=v.split('charset=')[1]
elifk=='plural-forms':
v=v.split(';')
plural=v[1].split('plural=')[1]
self.plural=c2py(plural)
# Note: we unconditionally convert both msgids and msgstrs to
# Unicode using the character encoding specified in the charset
# parameter of the Content-Type header. The gettext documentation
# strongly encourages msgids to be us-ascii, but some applications
# require alternative encodings (e.g. Zope's ZCML and ZPT). For
# traditional gettext applications, the msgid conversion will
# cause no problems since us-ascii should always be a subset of
# the charset encoding. We may want to fall back to 8-bit msgids
# if the Unicode conversion fails.
charset=self._charsetor'ascii'
ifb'\x00'inmsg:
# Plural forms
msgid1, msgid2=msg.split(b'\x00')
tmsg=tmsg.split(b'\x00')
msgid1=str(msgid1, charset)
fori, xinenumerate(tmsg):
catalog[(msgid1, i)] =str(x, charset)
else:
catalog[str(msg, charset)] =str(tmsg, charset)
# advance to next entry in the seek tables
masteridx+=8
transidx+=8

deflgettext(self, message):
missing=object()
tmsg=self._catalog.get(message, missing)
iftmsgismissing:
ifself._fallback:
returnself._fallback.lgettext(message)
tmsg=message
ifself._output_charset:
returntmsg.encode(self._output_charset)
returntmsg.encode(locale.getpreferredencoding())

deflngettext(self, msgid1, msgid2, n):
try:
tmsg=self._catalog[(msgid1, self.plural(n))]
exceptKeyError:
ifself._fallback:
returnself._fallback.lngettext(msgid1, msgid2, n)
ifn==1:
tmsg=msgid1
else:
tmsg=msgid2
ifself._output_charset:
returntmsg.encode(self._output_charset)
returntmsg.encode(locale.getpreferredencoding())

defgettext(self, message):
missing=object()
tmsg=self._catalog.get(message, missing)
iftmsgismissing:
ifself._fallback:
returnself._fallback.gettext(message)
returnmessage
returntmsg

defngettext(self, msgid1, msgid2, n):
try:
tmsg=self._catalog[(msgid1, self.plural(n))]
exceptKeyError:
ifself._fallback:
returnself._fallback.ngettext(msgid1, msgid2, n)
ifn==1:
tmsg=msgid1
else:
tmsg=msgid2
returntmsg


# Locate a .mo file using the gettext strategy
deffind(domain, localedir=None, languages=None, all=False):
# Get some reasonable defaults for arguments that were not supplied
iflocaledirisNone:
localedir=_default_localedir
iflanguagesisNone:
languages= []
forenvarin ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
val=os.environ.get(envar)
ifval:
languages=val.split(':')
break
if'C'notinlanguages:
languages.append('C')
# now normalize and expand the languages
nelangs= []
forlanginlanguages:
fornelangin_expand_lang(lang):
ifnelangnotinnelangs:
nelangs.append(nelang)
# select a language
ifall:
result= []
else:
result=None
forlanginnelangs:
iflang=='C':
break
mofile=os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo'%domain)
ifos.path.exists(mofile):
ifall:
result.append(mofile)
else:
returnmofile
returnresult



# a mapping between absolute .mo file path and Translation object
_translations= {}

deftranslation(domain, localedir=None, languages=None,
class_=None, fallback=False, codeset=None):
ifclass_isNone:
class_=GNUTranslations
mofiles=find(domain, localedir, languages, all=True)
ifnotmofiles:
iffallback:
returnNullTranslations()
fromerrnoimportENOENT
raiseFileNotFoundError(ENOENT,
'No translation file found for domain', domain)
# Avoid opening, reading, and parsing the .mo file after it's been done
# once.
result=None
formofileinmofiles:
key= (class_, os.path.abspath(mofile))
t=_translations.get(key)
iftisNone:
withopen(mofile, 'rb') asfp:
t=_translations.setdefault(key, class_(fp))
# Copy the translation object to allow setting fallbacks and
# output charset. All other instance data is shared with the
# cached object.
# Delay copy import for speeding up gettext import when .mo files
# are not used.
importcopy
t=copy.copy(t)
ifcodeset:
t.set_output_charset(codeset)
ifresultisNone:
result=t
else:
result.add_fallback(t)
returnresult


definstall(domain, localedir=None, codeset=None, names=None):
t=translation(domain, localedir, fallback=True, codeset=codeset)
t.install(names)



# a mapping b/w domains and locale directories
_localedirs= {}
# a mapping b/w domains and codesets
_localecodesets= {}
# current global domain, `messages' used for compatibility w/ GNU gettext
_current_domain='messages'


deftextdomain(domain=None):
global_current_domain
ifdomainisnotNone:
_current_domain=domain
return_current_domain


defbindtextdomain(domain, localedir=None):
global_localedirs
iflocaledirisnotNone:
_localedirs[domain] =localedir
return_localedirs.get(domain, _default_localedir)


defbind_textdomain_codeset(domain, codeset=None):
global_localecodesets
ifcodesetisnotNone:
_localecodesets[domain] =codeset
return_localecodesets.get(domain)


defdgettext(domain, message):
try:
t=translation(domain, _localedirs.get(domain, None),
codeset=_localecodesets.get(domain))
exceptOSError:
returnmessage
returnt.gettext(message)

defldgettext(domain, message):
codeset=_localecodesets.get(domain)
try:
t=translation(domain, _localedirs.get(domain, None), codeset=codeset)
exceptOSError:
returnmessage.encode(codesetorlocale.getpreferredencoding())
returnt.lgettext(message)

defdngettext(domain, msgid1, msgid2, n):
try:
t=translation(domain, _localedirs.get(domain, None),
codeset=_localecodesets.get(domain))
exceptOSError:
ifn==1:
returnmsgid1
else:
returnmsgid2
returnt.ngettext(msgid1, msgid2, n)

defldngettext(domain, msgid1, msgid2, n):
codeset=_localecodesets.get(domain)
try:
t=translation(domain, _localedirs.get(domain, None), codeset=codeset)
exceptOSError:
ifn==1:
tmsg=msgid1
else:
tmsg=msgid2
returntmsg.encode(codesetorlocale.getpreferredencoding())
returnt.lngettext(msgid1, msgid2, n)

defgettext(message):
returndgettext(_current_domain, message)

deflgettext(message):
returnldgettext(_current_domain, message)

defngettext(msgid1, msgid2, n):
returndngettext(_current_domain, msgid1, msgid2, n)

deflngettext(msgid1, msgid2, n):
returnldngettext(_current_domain, msgid1, msgid2, n)

# dcgettext() has been deemed unnecessary and is not implemented.

# James Henstridge's Catalog constructor from GNOME gettext. Documented usage
# was:
#
# import gettext
# cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
# _ = cat.gettext
# print _('Hello World')

# The resulting catalog object currently don't support access through a
# dictionary API, which was supported (but apparently unused) in GNOME
# gettext.

Catalog=translation