web2py.gluon.languages

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 This file is part of the web2py Web Framework 6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 8 """ 9 10 import os 11 import re 12 import cgi 13 import portalocker 14 import logging 15 import marshal 16 import copy_reg 17 from fileutils import listdir 18 import settings 19 from cfs import getcfs 20 21 __all__ = ['translator', 'findT', 'update_all_languages'] 22 23 is_gae = settings.global_settings.web2py_runtime_gae 24 25 # pattern to find T(blah blah blah) expressions 26 27 PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\ 28 + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\ 29 + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\ 30 + r'(?:"(?:[^"\\]|\\.)*"))' 31 32 regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL) 33 34 # patter for a valid accept_language 35 36 regex_language = \ 37 re.compile('^[a-zA-Z]{2}(\-[a-zA-Z]{2})?(\-[a-zA-Z]+)?$') 38 39

40 -def read_dict_aux(filename):

41 fp = open(filename, 'r') 42 portalocker.lock(fp, portalocker.LOCK_SH) 43 lang_text = fp.read().replace('\r\n', '\n') 44 portalocker.unlock(fp) 45 fp.close() 46 if not lang_text.strip(): 47 return {} 48 try: 49 return eval(lang_text) 50 except: 51 logging.error('Syntax error in %s' % filename) 52 return {}

53

54 -def read_dict(filename):

55 return getcfs('language:%s'%filename,filename, 56 lambda filename=filename:read_dict_aux(filename))

57

58 -def utf8_repr(s):

59 r''' # note that we use raw strings to avoid having to use double back slashes below 60 61 utf8_repr() works same as repr() when processing ascii string 62 >>> utf8_repr('abc') == utf8_repr("abc") == repr('abc') == repr("abc") == "'abc'" 63 True 64 >>> utf8_repr('a"b"c') == repr('a"b"c') == '\'a"b"c\'' 65 True 66 >>> utf8_repr("a'b'c") == repr("a'b'c") == '"a\'b\'c"' 67 True 68 >>> utf8_repr('a\'b"c') == repr('a\'b"c') == utf8_repr("a'b\"c") == repr("a'b\"c") == '\'a\\\'b"c\'' 69 True 70 >>> utf8_repr('a\r\nb') == repr('a\r\nb') == "'a\\r\\nb'" # Test for \r, \n 71 True 72 73 Unlike repr(), utf8_repr() remains utf8 content when processing utf8 string 74 >>> utf8_repr('中文字') == utf8_repr("中文字") == "'中文字'" != repr('中文字') 75 True 76 >>> utf8_repr('中"文"字') == "'中\"文\"字'" != repr('中"文"字') 77 True 78 >>> utf8_repr("中'文'字") == '"中\'文\'字"' != repr("中'文'字") 79 True 80 >>> utf8_repr('中\'文"字') == utf8_repr("中'文\"字") == '\'中\\\'文"字\'' != repr('中\'文"字') == repr("中'文\"字") 81 True 82 >>> utf8_repr('中\r\n文') == "'中\\r\\n文'" != repr('中\r\n文') # Test for \r, \n 83 True 84 ''' 85 if (s.find("'") >= 0) and (s.find('"') < 0): # only single quote exists 86 s = ''.join(['"', s, '"']) # s = ''.join(['"', s.replace('"','\\"'), '"']) 87 else: 88 s = ''.join(["'", s.replace("'","\\'"), "'"]) 89 return s.replace("\n","\\n").replace("\r","\\r")

90 91

92 -def write_dict(filename, contents):

93 try: 94 fp = open(filename, 'w') 95 except IOError: 96 logging.error('Unable to write to file %s' % filename) 97 return 98 portalocker.lock(fp, portalocker.LOCK_EX) 99 fp.write('# coding: utf8\n{\n') 100 for key in sorted(contents): 101 fp.write('%s: %s,\n' % (utf8_repr(key), utf8_repr(contents[key]))) 102 fp.write('}\n') 103 portalocker.unlock(fp) 104 fp.close()

105 106

107 -class lazyT(object):

108 109 """ 110 never to be called explicitly, returned by translator.__call__ 111 """ 112 113 m = None 114 s = None 115 T = None 116

117 - def __init__( 118 self, 119 message, 120 symbols = {}, 121 T = None, 122 ):

123 self.m = message 124 self.s = symbols 125 self.T = T

126

127 - def __repr__(self):

128 return "<lazyT %s>" % (repr(str(self.m)), )

129

130 - def __str__(self):

131 return self.T.translate(self.m, self.s)

132

133 - def __eq__(self, other):

134 return self.T.translate(self.m, self.s) == other

135

136 - def __ne__(self, other):

137 return self.T.translate(self.m, self.s) != other

138

139 - def __add__(self, other):

140 return '%s%s' % (self, other)

141

142 - def __radd__(self, other):

143 return '%s%s' % (other, self)

144

145 - def __cmp__(self,other):

146 return cmp(str(self),str(other))

147

148 - def __hash__(self):

149 return hash(str(self))

150

151 - def __getattr__(self, name):

152 return getattr(str(self),name)

153

154 - def __getitem__(self, i):

155 return str(self)[i]

156

157 - def __getslice__(self, i, j):

158 return str(self)[i:j]

159

160 - def __iter__(self):

161 for c in str(self): yield c

162

163 - def __len__(self):

164 return len(str(self))

165

166 - def xml(self):

167 return cgi.escape(str(self))

168

169 - def encode(self, *a, **b):

170 return str(self).encode(*a, **b)

171

172 - def decode(self, *a, **b):

173 return str(self).decode(*a, **b)

174

175 - def read(self):

176 return str(self)

177

178 - def __mod__(self, symbols):

179 return self.T.translate(self.m, symbols)

180 181

182 -class translator(object):

183 184 """ 185 this class is instantiated by gluon.compileapp.build_environment 186 as the T object 187 188 :: 189 190 T.force(None) # turns off translation 191 T.force('fr, it') # forces web2py to translate using fr.py or it.py 192 193 T(\"Hello World\") # translates \"Hello World\" using the selected file 194 195 notice 1: there is no need to force since, by default, T uses 196 accept_language to determine a translation file. 197 198 notice 2: en and en-en are considered different languages! 199 """ 200

201 - def __init__(self, request):

202 self.request = request 203 self.folder = request.folder 204 self.current_languages = ['en'] 205 self.accepted_language = None 206 self.language_file = None 207 self.http_accept_language = request.env.http_accept_language 208 self.requested_languages = self.force(self.http_accept_language) 209 self.lazy = True 210 self.otherTs = {}

211

212 - def get_possible_languages(self):

213 possible_languages = self.current_languages 214 file_ending = re.compile("\.py$") 215 for langfile in os.listdir(os.path.join(self.folder,'languages')): 216 if file_ending.search(langfile): 217 possible_languages.append(file_ending.sub('',langfile)) 218 return possible_languages

219

220 - def set_current_languages(self, *languages):

221 if len(languages) == 1 and isinstance(languages[0], (tuple, list)): 222 languages = languages[0] 223 self.current_languages = languages 224 self.force(self.http_accept_language)

225

226 - def force(self, *languages):

227 if not languages or languages[0] == None: 228 languages = [] 229 if len(languages) == 1 and isinstance(languages[0], (str, unicode)): 230 languages = languages[0] 231 if languages: 232 if isinstance(languages, (str, unicode)): 233 accept_languages = languages.split(';') 234 languages = [] 235 [languages.extend(al.split(',')) for al in accept_languages] 236 languages = [item.strip().lower() for item in languages \ 237 if regex_language.match(item.strip())] 238 239 for language in languages: 240 if language in self.current_languages: 241 self.accepted_language = language 242 break 243 filename = os.path.join(self.folder, 'languages/', language + '.py') 244 if os.path.exists(filename): 245 self.accepted_language = language 246 self.language_file = filename 247 self.t = read_dict(filename) 248 return languages 249 self.language_file = None 250 self.t = {} # ## no language by default 251 return languages

252

253 - def __call__(self, message, symbols={},language=None):

254 if not language: 255 if self.lazy: 256 return lazyT(message, symbols, self) 257 else: 258 return self.translate(message, symbols) 259 else: 260 try: 261 otherT = self.otherTs[language] 262 except KeyError: 263 otherT = self.otherTs[language] = translator(self.request) 264 otherT.force(language) 265 return otherT(message,symbols)

266

267 - def translate(self, message, symbols):

268 """ 269 user ## to add a comment into a translation string 270 the comment can be useful do discriminate different possible 271 translations for the same string (for example different locations) 272 273 T(' hello world ') -> ' hello world ' 274 T(' hello world ## token') -> 'hello world' 275 T('hello ## world ## token') -> 'hello ## world' 276 277 the ## notation is ignored in multiline strings and strings that 278 start with ##. this is to allow markmin syntax to be translated 279 """ 280 if not message.startswith('#') and not '\n' in message: 281 tokens = message.rsplit('##', 1) 282 else: 283 # this allows markmin syntax in translations 284 tokens = [message] 285 if len(tokens) == 2: 286 tokens[0] = tokens[0].strip() 287 message = tokens[0] + '##' + tokens[1].strip() 288 mt = self.t.get(message, None) 289 if mt == None: 290 self.t[message] = mt = tokens[0] 291 if self.language_file and not is_gae: 292 write_dict(self.language_file, self.t) 293 if symbols or symbols == 0: 294 return mt % symbols 295 return mt

296 297

298 -def findT(path, language='en-us'):

299 """ 300 must be run by the admin app 301 """ 302 filename = os.path.join(path, 'languages', '%s.py' % language) 303 sentences = read_dict(filename) 304 mp = os.path.join(path, 'models') 305 cp = os.path.join(path, 'controllers') 306 vp = os.path.join(path, 'views') 307 for file in listdir(mp, '.+\.py', 0) + listdir(cp, '.+\.py', 0)\ 308 + listdir(vp, '.+\.html', 0): 309 fp = open(file, 'r') 310 portalocker.lock(fp, portalocker.LOCK_SH) 311 data = fp.read() 312 portalocker.unlock(fp) 313 fp.close() 314 items = regex_translate.findall(data) 315 for item in items: 316 try: 317 message = eval(item) 318 if not message.startswith('#') and not '\n' in message: 319 tokens = message.rsplit('##', 1) 320 else: 321 # this allows markmin syntax in translations 322 tokens = [message] 323 if len(tokens) == 2: 324 message = tokens[0].strip() + '##' + tokens[1].strip() 325 if message and not message in sentences: 326 sentences[message] = message 327 except: 328 pass 329 write_dict(filename, sentences)

330 331 ### important to allow safe session.flash=T(....)

332 -def lazyT_unpickle(data):

333 return marshal.loads(data)

334 -def lazyT_pickle(data):

335 return lazyT_unpickle, (marshal.dumps(str(data)),)

336 copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle) 337

338 -def update_all_languages(application_path):

339 path = os.path.join(application_path, 'languages/') 340 for language in listdir(path, '^\w+(\-\w+)?\.py$'): 341 findT(application_path, language[:-3])

342 343 344 if __name__ == '__main__': 345 import doctest 346 doctest.testmod() 347

Source Code for Module web2py.gluon.languages