Package web2py :: Package gluon :: Module languages |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.languages

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  This file is part of the web2py Web Framework
      6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
      8  """
      9
     10  import os
     11  import re
     12  import cgi
     13  import portalocker
     14  import logging
     15  import marshal
     16  import copy_reg
     17  from fileutils import listdir
     18  import settings
     19  from cfs import getcfs
     20
     21  __all__ = ['translator', 'findT', 'update_all_languages']
     22
     23  is_gae = settings.global_settings.web2py_runtime_gae
     24
     25  # pattern to find T(blah blah blah) expressions
     26
     27  PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\
     28       + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\
     29       + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\
     30       + r'(?:"(?:[^"\\]|\\.)*"))'
     31
     32  regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL)
     33
     34  # patter for a valid accept_language
     35
     36  regex_language = \
     37      re.compile('^[a-zA-Z]{2}(\-[a-zA-Z]{2})?(\-[a-zA-Z]+)?$')
     38
     39
     40 -def read_dict_aux(filename):
     41      fp = open(filename, 'r')
     42      portalocker.lock(fp, portalocker.LOCK_SH)
     43      lang_text = fp.read().replace('\r\n', '\n')
     44      portalocker.unlock(fp)
     45      fp.close()
     46      if not lang_text.strip():
     47          return {}
     48      try:
     49          return eval(lang_text)
     50      except:
     51          logging.error('Syntax error in %s' % filename)
     52          return {}
     53
     54 -def read_dict(filename):
     55      return getcfs('language:%s'%filename,filename,
     56                    lambda filename=filename:read_dict_aux(filename))
     57
     58 -def utf8_repr(s):
     59      r''' # note that we use raw strings to avoid having to use double back slashes below
     60
     61      utf8_repr() works same as repr() when processing ascii string
     62      >>> utf8_repr('abc') == utf8_repr("abc") == repr('abc') == repr("abc") == "'abc'"
     63      True
     64      >>> utf8_repr('a"b"c') == repr('a"b"c') == '\'a"b"c\''
     65      True
     66      >>> utf8_repr("a'b'c") == repr("a'b'c") == '"a\'b\'c"'
     67      True
     68      >>> utf8_repr('a\'b"c') == repr('a\'b"c') == utf8_repr("a'b\"c") == repr("a'b\"c") == '\'a\\\'b"c\''
     69      True
     70      >>> utf8_repr('a\r\nb') == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n
     71      True
     72
     73      Unlike repr(), utf8_repr() remains utf8 content when processing utf8 string
     74      >>> utf8_repr('中文字') == utf8_repr("中文字") == "'中文字'" != repr('中文字')
     75      True
     76      >>> utf8_repr('中"文"字') == "'中\"文\"字'" != repr('中"文"字')
     77      True
     78      >>> utf8_repr("中'文'字") == '"中\'文\'字"' != repr("中'文'字")
     79      True
     80      >>> utf8_repr('中\'文"字') == utf8_repr("中'文\"字") == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字")
     81      True
     82      >>> utf8_repr('中\r\n文') == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n
     83      True
     84      '''
     85      if (s.find("'") >= 0) and (s.find('"') < 0): # only single quote exists
     86          s = ''.join(['"', s, '"']) # s = ''.join(['"', s.replace('"','\\"'), '"'])
     87      else:
     88          s = ''.join(["'", s.replace("'","\\'"), "'"])
     89      return s.replace("\n","\\n").replace("\r","\\r")
     90
     91
     92 -def write_dict(filename, contents):
     93      try:
     94          fp = open(filename, 'w')
     95      except IOError:
     96          logging.error('Unable to write to file %s' % filename)
     97          return
     98      portalocker.lock(fp, portalocker.LOCK_EX)
     99      fp.write('# coding: utf8\n{\n')
    100      for key in sorted(contents):
    101          fp.write('%s: %s,\n' % (utf8_repr(key), utf8_repr(contents[key])))
    102      fp.write('}\n')
    103      portalocker.unlock(fp)
    104      fp.close()
    105
    106
    107 -class lazyT(object):
    108
    109      """
    110      never to be called explicitly, returned by translator.__call__
    111      """
    112
    113      m = None
    114      s = None
    115      T = None
    116
    117 -    def __init__(
    118          self,
    119          message,
    120          symbols = {},
    121          T = None,
    122          ):
    123          self.m = message
    124          self.s = symbols
    125          self.T = T
    126
    127 -    def __repr__(self):
    128          return "<lazyT %s>" % (repr(str(self.m)), )
    129
    130 -    def __str__(self):
    131          return self.T.translate(self.m, self.s)
    132
    133 -    def __eq__(self, other):
    134          return self.T.translate(self.m, self.s) == other
    135
    136 -    def __ne__(self, other):
    137          return self.T.translate(self.m, self.s) != other
    138
    139 -    def __add__(self, other):
    140          return '%s%s' % (self, other)
    141
    142 -    def __radd__(self, other):
    143          return '%s%s' % (other, self)
    144
    145 -    def __cmp__(self,other):
    146          return cmp(str(self),str(other))
    147
    148 -    def __hash__(self):
    149          return hash(str(self))
    150
    151 -    def __getattr__(self, name):
    152          return getattr(str(self),name)
    153
    154 -    def __getitem__(self, i):
    155          return str(self)[i]
    156
    157 -    def __getslice__(self, i, j):
    158          return str(self)[i:j]
    159
    160 -    def __iter__(self):
    161          for c in str(self): yield c
    162
    163 -    def __len__(self):
    164          return len(str(self))
    165
    166 -    def xml(self):
    167          return cgi.escape(str(self))
    168
    169 -    def encode(self, *a, **b):
    170          return str(self).encode(*a, **b)
    171
    172 -    def decode(self, *a, **b):
    173          return str(self).decode(*a, **b)
    174
    175 -    def read(self):
    176          return str(self)
    177
    178 -    def __mod__(self, symbols):
    179          return self.T.translate(self.m, symbols)
    180
    181
    182 -class translator(object):
    183
    184      """
    185      this class is instantiated by gluon.compileapp.build_environment
    186      as the T object
    187
    188      ::
    189
    190          T.force(None) # turns off translation
    191          T.force('fr, it') # forces web2py to translate using fr.py or it.py
    192
    193          T(\"Hello World\") # translates \"Hello World\" using the selected file
    194
    195      notice 1: there is no need to force since, by default, T uses
    196      accept_language to determine a translation file.
    197
    198      notice 2: en and en-en are considered different languages!
    199      """
    200
    201 -    def __init__(self, request):
    202          self.request = request
    203          self.folder = request.folder
    204          self.current_languages = ['en']
    205          self.accepted_language = None
    206          self.language_file = None
    207          self.http_accept_language = request.env.http_accept_language
    208          self.requested_languages = self.force(self.http_accept_language)
    209          self.lazy = True
    210          self.otherTs = {}
    211
    212 -    def get_possible_languages(self):
    213          possible_languages = self.current_languages
    214          file_ending = re.compile("\.py$")
    215          for langfile in os.listdir(os.path.join(self.folder,'languages')):
    216              if file_ending.search(langfile):
    217                  possible_languages.append(file_ending.sub('',langfile))
    218          return possible_languages
    219
    220 -    def set_current_languages(self, *languages):
    221          if len(languages) == 1 and isinstance(languages[0], (tuple, list)):
    222              languages = languages[0]
    223          self.current_languages = languages
    224          self.force(self.http_accept_language)
    225
    226 -    def force(self, *languages):
    227          if not languages or languages[0] == None:
    228              languages = []
    229          if len(languages) == 1 and isinstance(languages[0], (str, unicode)):
    230              languages = languages[0]
    231          if languages:
    232              if isinstance(languages, (str, unicode)):
    233                  accept_languages = languages.split(';')
    234                  languages = []
    235                  [languages.extend(al.split(',')) for al in accept_languages]
    236                  languages = [item.strip().lower() for item in languages \
    237                                   if regex_language.match(item.strip())]
    238
    239              for language in languages:
    240                  if language in self.current_languages:
    241                      self.accepted_language = language
    242                      break
    243                  filename = os.path.join(self.folder, 'languages/', language + '.py')
    244                  if os.path.exists(filename):
    245                      self.accepted_language = language
    246                      self.language_file = filename
    247                      self.t = read_dict(filename)
    248                      return languages
    249          self.language_file = None
    250          self.t = {}  # ## no language by default
    251          return languages
    252
    253 -    def __call__(self, message, symbols={},language=None):
    254          if not language:
    255              if self.lazy:
    256                  return lazyT(message, symbols, self)
    257              else:
    258                  return self.translate(message, symbols)
    259          else:
    260              try:
    261                  otherT = self.otherTs[language]
    262              except KeyError:
    263                  otherT = self.otherTs[language] = translator(self.request)
    264                  otherT.force(language)
    265              return otherT(message,symbols)
    266
    267 -    def translate(self, message, symbols):
    268          """
    269          user ## to add a comment into a translation string
    270          the comment can be useful do discriminate different possible
    271          translations for the same string (for example different locations)
    272
    273          T(' hello world ') -> ' hello world '
    274          T(' hello world ## token') -> 'hello world'
    275          T('hello ## world ## token') -> 'hello ## world'
    276
    277          the ## notation is ignored in multiline strings and strings that
    278          start with ##. this is to allow markmin syntax to be translated
    279          """
    280          if not message.startswith('#') and not '\n' in message:
    281              tokens = message.rsplit('##', 1)
    282          else:
    283              # this allows markmin syntax in translations
    284              tokens = [message]
    285          if len(tokens) == 2:
    286              tokens[0] = tokens[0].strip()
    287              message = tokens[0] + '##' + tokens[1].strip()
    288          mt = self.t.get(message, None)
    289          if mt == None:
    290              self.t[message] = mt = tokens[0]
    291              if self.language_file and not is_gae:
    292                  write_dict(self.language_file, self.t)
    293          if symbols or symbols == 0:
    294              return mt % symbols
    295          return mt
    296
    297
    298 -def findT(path, language='en-us'):
    299      """
    300      must be run by the admin app
    301      """
    302      filename = os.path.join(path, 'languages', '%s.py' % language)
    303      sentences = read_dict(filename)
    304      mp = os.path.join(path, 'models')
    305      cp = os.path.join(path, 'controllers')
    306      vp = os.path.join(path, 'views')
    307      for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\
    308           + listdir(vp, '.+\.html', 0):
    309          fp = open(file, 'r')
    310          portalocker.lock(fp, portalocker.LOCK_SH)
    311          data = fp.read()
    312          portalocker.unlock(fp)
    313          fp.close()
    314          items = regex_translate.findall(data)
    315          for item in items:
    316              try:
    317                  message = eval(item)
    318                  if not message.startswith('#') and not '\n' in message:
    319                      tokens = message.rsplit('##', 1)
    320                  else:
    321                      # this allows markmin syntax in translations
    322                      tokens = [message]
    323                  if len(tokens) == 2:
    324                      message = tokens[0].strip() + '##' + tokens[1].strip()
    325                  if message and not message in sentences:
    326                      sentences[message] = message
    327              except:
    328                  pass
    329      write_dict(filename, sentences)
    330
    331  ### important to allow safe session.flash=T(....)
    332 -def lazyT_unpickle(data):
    333      return marshal.loads(data)
    334 -def lazyT_pickle(data):
    335      return lazyT_unpickle, (marshal.dumps(str(data)),)
    336  copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle)
    337
    338 -def update_all_languages(application_path):
    339      path = os.path.join(application_path, 'languages/')
    340      for language in listdir(path, '^\w+(\-\w+)?\.py$'):
    341          findT(application_path, language[:-3])
    342
    343
    344  if __name__ == '__main__':
    345      import doctest
    346      doctest.testmod()
    347