Package web2py :: Package gluon :: Module fileutils
[hide private]
[frames] | no frames]

Source Code for Module web2py.gluon.fileutils

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  """ 
  5  This file is part of the web2py Web Framework 
  6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
  7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 
  8  """ 
  9   
 10  import storage 
 11  import os 
 12  import re 
 13  import tarfile 
 14  import glob 
 15  import time 
 16  from http import HTTP 
 17  from gzip import open as gzopen 
 18  from settings import global_settings 
 19   
 20   
 21  __all__ = [ 
 22      'read_file', 
 23      'write_file', 
 24      'readlines_file', 
 25      'up', 
 26      'abspath', 
 27      'mktree', 
 28      'listdir', 
 29      'recursive_unlink', 
 30      'cleanpath', 
 31      'tar', 
 32      'untar', 
 33      'tar_compiled', 
 34      'get_session', 
 35      'check_credentials', 
 36      'w2p_pack', 
 37      'w2p_unpack', 
 38      'w2p_pack_plugin', 
 39      'w2p_unpack_plugin', 
 40      'fix_newlines', 
 41      'make_fake_file_like_object', 
 42      ] 
 43   
44 -def read_file(filename, mode='r'):
45 "returns content from filename, making sure to close the file explicitly on exit." 46 f = open(filename, mode) 47 try: 48 return f.read() 49 finally: 50 f.close()
51
52 -def write_file(filename, value, mode='w'):
53 "writes <value> to filename, making sure to close the file explicitly on exit." 54 f = open(filename, mode) 55 try: 56 return f.write(value) 57 finally: 58 f.close()
59
60 -def readlines_file(filename, mode='r'):
61 "applies .split('\n') to the output of read_file()" 62 return read_file(filename, mode).split('\n')
63
64 -def abspath(*relpath, **base):
65 "convert relative path to absolute path based (by default) on applications_parent" 66 path = os.path.join(*relpath) 67 gluon = base.get('gluon', False) 68 if os.path.isabs(path): 69 return path 70 if gluon: 71 return os.path.join(global_settings.gluon_parent, path) 72 return os.path.join(global_settings.applications_parent, path)
73 74
75 -def mktree(path):
76 head,tail =os.path.split(path) 77 if head: 78 if tail: mktree(head) 79 if not os.path.exists(head): 80 os.mkdir(head)
81
82 -def listdir( 83 path, 84 expression='^.+$', 85 drop=True, 86 add_dirs=False, 87 sort=True, 88 ):
89 """ 90 like os.listdir() but you can specify a regex pattern to filter files. 91 if add_dirs is True, the returned items will have the full path. 92 """ 93 if path[-1:] != os.path.sep: 94 path = path + os.path.sep 95 if drop: 96 n = len(path) 97 else: 98 n = 0 99 regex = re.compile(expression) 100 items = [] 101 for (root, dirs, files) in os.walk(path, topdown=True): 102 for dir in dirs[:]: 103 if dir.startswith('.'): 104 dirs.remove(dir) 105 if add_dirs: 106 items.append(root[n:]) 107 for file in sorted(files): 108 if regex.match(file) and not file.startswith('.'): 109 items.append(os.path.join(root, file)[n:]) 110 if sort: 111 return sorted(items) 112 else: 113 return items
114 115 123 124
125 -def cleanpath(path):
126 """ 127 turns any expression/path into a valid filename. replaces / with _ and 128 removes special characters. 129 """ 130 131 items = path.split('.') 132 if len(items) > 1: 133 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.' 134 + ''.join(items[-1:])) 135 else: 136 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:])) 137 return path
138 139
140 -def _extractall(filename, path='.', members=None):
141 if not hasattr(tarfile.TarFile, 'extractall'): 142 from tarfile import ExtractError 143 144 class TarFile(tarfile.TarFile): 145 146 def extractall(self, path='.', members=None): 147 """Extract all members from the archive to the current working 148 directory and set owner, modification time and permissions on 149 directories afterwards. `path' specifies a different directory 150 to extract to. `members' is optional and must be a subset of the 151 list returned by getmembers(). 152 """ 153 154 directories = [] 155 if members is None: 156 members = self 157 for tarinfo in members: 158 if tarinfo.isdir(): 159 160 # Extract directory with a safe mode, so that 161 # all files below can be extracted as well. 162 163 try: 164 os.makedirs(os.path.join(path, 165 tarinfo.name), 0777) 166 except EnvironmentError: 167 pass 168 directories.append(tarinfo) 169 else: 170 self.extract(tarinfo, path) 171 172 # Reverse sort directories. 173 174 directories.sort(lambda a, b: cmp(a.name, b.name)) 175 directories.reverse() 176 177 # Set correct owner, mtime and filemode on directories. 178 179 for tarinfo in directories: 180 path = os.path.join(path, tarinfo.name) 181 try: 182 self.chown(tarinfo, path) 183 self.utime(tarinfo, path) 184 self.chmod(tarinfo, path) 185 except ExtractError, e: 186 if self.errorlevel > 1: 187 raise 188 else: 189 self._dbg(1, 'tarfile: %s' % e)
190 191 192 _cls = TarFile 193 else: 194 _cls = tarfile.TarFile 195 196 tar = _cls(filename, 'r') 197 ret = tar.extractall(path, members) 198 tar.close() 199 return ret 200
201 -def tar(file, dir, expression='^.+$'):
202 """ 203 tars dir into file, only tars file that match expression 204 """ 205 206 tar = tarfile.TarFile(file, 'w') 207 try: 208 for file in listdir(dir, expression, add_dirs=True): 209 tar.add(os.path.join(dir, file), file, False) 210 finally: 211 tar.close()
212
213 -def untar(file, dir):
214 """ 215 untar file into dir 216 """ 217 218 _extractall(file, dir)
219 220
221 -def w2p_pack(filename, path, compiled=False):
222 filename = abspath(filename) 223 path = abspath(path) 224 tarname = filename + '.tar' 225 if compiled: 226 tar_compiled(tarname, path, '^[\w\.\-]+$') 227 else: 228 tar(tarname, path, '^[\w\.\-]+$') 229 w2pfp = gzopen(filename, 'wb') 230 tarfp = open(tarname, 'rb') 231 w2pfp.write(tarfp.read()) 232 w2pfp.close() 233 tarfp.close() 234 os.unlink(tarname)
235
236 -def w2p_unpack(filename, path, delete_tar=True):
237 filename = abspath(filename) 238 path = abspath(path) 239 if filename[-4:] == '.w2p' or filename[-3:] == '.gz': 240 if filename[-4:] == '.w2p': 241 tarname = filename[:-4] + '.tar' 242 else: 243 tarname = filename[:-3] + '.tar' 244 fgzipped = gzopen(filename, 'rb') 245 tarfile = open(tarname, 'wb') 246 tarfile.write(fgzipped.read()) 247 tarfile.close() 248 fgzipped.close() 249 else: 250 tarname = filename 251 untar(tarname, path) 252 if delete_tar: 253 os.unlink(tarname)
254 255
256 -def w2p_pack_plugin(filename, path, plugin_name):
257 """Pack the given plugin into a w2p file. 258 Will match files at: 259 <path>/*/plugin_[name].* 260 <path>/*/plugin_[name]/* 261 """ 262 filename = abspath(filename) 263 path = abspath(path) 264 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): 265 raise Exception, "Not a web2py plugin name" 266 plugin_tarball = tarfile.open(filename, 'w:gz') 267 try: 268 app_dir = path 269 while app_dir[-1]=='/': 270 app_dir = app_dir[:-1] 271 files1=glob.glob(os.path.join(app_dir,'*/plugin_%s.*' % plugin_name)) 272 files2=glob.glob(os.path.join(app_dir,'*/plugin_%s/*' % plugin_name)) 273 for file in files1+files2: 274 plugin_tarball.add(file, arcname=file[len(app_dir)+1:]) 275 finally: 276 plugin_tarball.close()
277 278
279 -def w2p_unpack_plugin(filename, path, delete_tar=True):
280 filename = abspath(filename) 281 path = abspath(path) 282 if not os.path.basename(filename).startswith('web2py.plugin.'): 283 raise Exception, "Not a web2py plugin" 284 w2p_unpack(filename,path,delete_tar)
285 286
287 -def tar_compiled(file, dir, expression='^.+$'):
288 """ 289 used to tar a compiled application. 290 the content of models, views, controllers is not stored in the tar file. 291 """ 292 293 tar = tarfile.TarFile(file, 'w') 294 for file in listdir(dir, expression, add_dirs=True): 295 filename = os.path.join(dir, file) 296 if os.path.islink(filename): 297 continue 298 if os.path.isfile(filename) and file[-4:] != '.pyc': 299 if file[:6] == 'models': 300 continue 301 if file[:5] == 'views': 302 continue 303 if file[:11] == 'controllers': 304 continue 305 if file[:7] == 'modules': 306 continue 307 tar.add(filename, file, False) 308 tar.close()
309
310 -def up(path):
311 return os.path.dirname(os.path.normpath(path))
312 313
314 -def get_session(request, other_application='admin'):
315 """ checks that user is authorized to access other_application""" 316 if request.application == other_application: 317 raise KeyError 318 try: 319 session_id = request.cookies['session_id_' + other_application].value 320 osession = storage.load_storage(os.path.join( 321 up(request.folder), other_application, 'sessions', session_id)) 322 except: 323 osession = storage.Storage() 324 return osession
325 326
327 -def check_credentials(request, other_application='admin', expiration = 60*60):
328 """ checks that user is authorized to access other_application""" 329 if request.env.web2py_runtime_gae: 330 from google.appengine.api import users 331 if users.is_current_user_admin(): 332 return True 333 else: 334 login_html = '<a href="%s">Sign in with your google account</a>.' \ 335 % users.create_login_url(request.env.path_info) 336 raise HTTP(200, '<html><body>%s</body></html>' % login_html) 337 else: 338 dt = time.time() - expiration 339 s = get_session(request, other_application) 340 return (s.authorized and s.last_time and s.last_time > dt)
341 342
343 -def fix_newlines(path):
344 regex = re.compile(r'''(\r 345 |\r| 346 )''') 347 for filename in listdir(path, '.*\.(py|html)$', drop=False): 348 rdata = read_file(filename, 'rb') 349 wdata = regex.sub('\n', rdata) 350 if wdata != rdata: 351 write_file(filename, wdata, 'wb')
352
353 -def copystream( 354 src, 355 dest, 356 size, 357 chunk_size=10 ** 5, 358 ):
359 """ 360 this is here because I think there is a bug in shutil.copyfileobj 361 """ 362 while size > 0: 363 if size < chunk_size: 364 data = src.read(size) 365 else: 366 data = src.read(chunk_size) 367 length = len(data) 368 if length > size: 369 (data, length) = (data[:size], size) 370 size -= length 371 if length == 0: 372 break 373 dest.write(data) 374 if length < chunk_size: 375 break 376 dest.seek(0) 377 return
378 379
380 -def make_fake_file_like_object():
381 class LogFile(object): 382 def write(self, value): 383 pass
384 def close(self): 385 pass 386 return LogFile() 387