Package web2py :: Package gluon :: Module compileapp |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.compileapp

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  This file is part of the web2py Web Framework
      6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
      8
      9  Functions required to execute app components
     10  ============================================
     11
     12  FOR INTERNAL USE ONLY
     13  """
     14
     15  import re
     16  import sys
     17  import fnmatch
     18  import os
     19  import copy
     20  import random
     21  import __builtin__
     22  from storage import Storage, List
     23  from template import parse_template
     24  from restricted import restricted, compile2
     25  from fileutils import mktree, listdir, read_file, write_file
     26  from myregex import regex_expose
     27  from languages import translator
     28  from dal import BaseAdapter, SQLDB, SQLField, DAL, Field
     29  from sqlhtml import SQLFORM, SQLTABLE
     30  from cache import Cache
     31  from globals import current
     32  import settings
     33  from cfs import getcfs
     34  import html
     35  import validators
     36  from http import HTTP, redirect
     37  import marshal
     38  import shutil
     39  import imp
     40  import logging
     41  logger = logging.getLogger("web2py")
     42  import rewrite
     43
     44  try:
     45      import py_compile
     46  except:
     47      logger.warning('unable to import py_compile')
     48
     49  is_gae = settings.global_settings.web2py_runtime_gae
     50  is_jython = settings.global_settings.is_jython = 'java' in sys.platform.lower() or hasattr(sys, 'JYTHON_JAR') or str(sys.copyright).find('Jython') > 0
     51
     52  TEST_CODE = \
     53      r"""
     54  def _TEST():
     55      import doctest, sys, cStringIO, types, cgi, gluon.fileutils
     56      if not gluon.fileutils.check_credentials(request):
     57          raise HTTP(401, web2py_error='invalid credentials')
     58      stdout = sys.stdout
     59      html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \
     60          % request.controller
     61      for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]):
     62          eval_key = eval(key)
     63          if type(eval_key) == types.FunctionType:
     64              number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)])
     65              if number_doctests>0:
     66                  sys.stdout = cStringIO.StringIO()
     67                  name = '%s/controllers/%s.py in %s.__doc__' \
     68                      % (request.folder, request.controller, key)
     69                  doctest.run_docstring_examples(eval_key,
     70                      globals(), False, name=name)
     71                  report = sys.stdout.getvalue().strip()
     72                  if report:
     73                      pf = 'failed'
     74                  else:
     75                      pf = 'passed'
     76                  html += '<h3 class="%s">Function %s [%s]</h3>\n' \
     77                      % (pf, key, pf)
     78                  if report:
     79                      html += CODE(report, language='web2py', \
     80                          link='/examples/global/vars/').xml()
     81                  html += '<br/>\n'
     82              else:
     83                  html += \
     84                      '<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \
     85                      % (key)
     86      response._vars = html
     87      sys.stdout = stdout
     88  _TEST()
     89  """
     90
     91 -class mybuiltin(object):
     92      """
     93      NOTE could simple use a dict and populate it,
     94      NOTE not sure if this changes things though if monkey patching import.....
     95      """
     96      #__builtins__
     97 -    def __getitem__(self, key):
     98          try:
     99              return getattr(__builtin__, key)
    100          except AttributeError:
    101              raise KeyError, key
    102 -    def __setitem__(self, key, value):
    103          setattr(self, key, value)
    104
    105 -class LoadFactory(object):
    106      """
    107      Attention: this helper is new and experimental
    108      """
    109 -    def __init__(self,environment):
    110          self.environment = environment
    111 -    def __call__(self, c=None, f='index', args=[], vars={},
    112                   extension=None, target=None,ajax=False,ajax_trap=False,
    113                   url=None,user_signature=False, content='loading...',**attr):
    114          import globals
    115          target = target or 'c'+str(random.random())[2:]
    116          attr['_id']=target
    117          request = self.environment['request']
    118          if not isinstance(vars,Storage):
    119              vars = Storage(vars)
    120          if '.' in f:
    121              f, extension = f.split('.',1)
    122          if url or ajax:
    123              url = url or html.URL(request.application, c, f, r=request,
    124                                    args=args, vars=vars, extension=extension,
    125                                    user_signature=user_signature)
    126              script = html.SCRIPT('web2py_component("%s","%s")' % (url, target),
    127                                   _type="text/javascript")
    128              return html.TAG[''](script, html.DIV(content,**attr))
    129          else:
    130              if not isinstance(args,(list,tuple)):
    131                  args = [args]
    132              c = c or request.controller
    133
    134              other_request = Storage()
    135              for key, value in request.items():
    136                  other_request[key] = value
    137              other_request['env'] = Storage()
    138              for key, value in request.env.items():
    139                  other_request.env['key'] = value
    140              other_request.controller = c
    141              other_request.function = f
    142              other_request.extension = extension or request.extension
    143              other_request.args = List(args)
    144              other_request.vars = vars
    145              other_request.get_vars = vars
    146              other_request.post_vars = Storage()
    147              other_response = globals.Response()
    148              other_request.env.path_info = '/' + \
    149                  '/'.join([request.application,c,f] + \
    150                               map(str, other_request.args))
    151              other_request.env.query_string = \
    152                  vars and html.URL(vars=vars).split('?')[1] or ''
    153              other_request.env.http_web2py_component_location = \
    154                  request.env.path_info
    155              other_request.cid = target
    156              other_request.env.http_web2py_component_element = target
    157              other_response.view = '%s/%s.%s' % (c,f, other_request.extension)
    158              other_environment = copy.copy(self.environment)
    159              other_response._view_environment = other_environment
    160              other_response.generic_patterns = \
    161                  copy.copy(current.response.generic_patterns)
    162              other_environment['request'] = other_request
    163              other_environment['response'] = other_response
    164
    165              ## some magic here because current are thread-locals
    166
    167              original_request, current.request = current.request, other_request
    168              original_response, current.response = current.response, other_response
    169              page = run_controller_in(c, f, other_environment)
    170              if isinstance(page, dict):
    171                  other_response._vars = page
    172                  for key in page:
    173                      other_response._view_environment[key] = page[key]
    174                  run_view_in(other_response._view_environment)
    175                  page = other_response.body.getvalue()
    176              current.request, current.response = original_request, original_response
    177              js = None
    178              if ajax_trap:
    179                  link = html.URL(request.application, c, f, r=request,
    180                                  args=args, vars=vars, extension=extension,
    181                                  user_signature=user_signature)
    182                  js = "web2py_trap_form('%s','%s');" % (link, target)
    183              script = js and html.SCRIPT(js,_type="text/javascript") or ''
    184              return html.TAG[''](html.DIV(html.XML(page),**attr),script)
    185
    186
    187 -def local_import_aux(name, force=False, app='welcome'):
    188      """
    189      In apps, instead of importing a local module
    190      (in applications/app/modules) with::
    191
    192         import a.b.c as d
    193
    194      you should do::
    195
    196         d = local_import('a.b.c')
    197
    198      or (to force a reload):
    199
    200         d = local_import('a.b.c', reload=True)
    201
    202      This prevents conflict between applications and un-necessary execs.
    203      It can be used to import any module, including regular Python modules.
    204      """
    205      items = name.replace('/','.')
    206      name = "applications.%s.modules.%s" % (app, items)
    207      module = __import__(name)
    208      for item in name.split(".")[1:]:
    209          module = getattr(module, item)
    210      if force:
    211          reload(module)
    212      return module
    213
    214
    215  """
    216  OLD IMPLEMENTATION:
    217      items = name.replace('/','.').split('.')
    218      filename, modulepath = items[-1], os.path.join(apath,'modules',*items[:-1])
    219      imp.acquire_lock()
    220      try:
    221          file=None
    222          (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path)
    223          if not path in sys.modules or reload:
    224              if is_gae:
    225                  module={}
    226                  execfile(path,{},module)
    227                  module=Storage(module)
    228              else:
    229                  module = imp.load_module(path,file,path,desc)
    230              sys.modules[path] = module
    231          else:
    232              module = sys.modules[path]
    233      except Exception, e:
    234          module = None
    235      if file:
    236          file.close()
    237      imp.release_lock()
    238      if not module:
    239          raise ImportError, "cannot find module %s in %s" % (filename, modulepath)
    240      return module
    241  """
    242
    243 -def build_environment(request, response, session, store_current=True):
    244      """
    245      Build the environment dictionary into which web2py files are executed.
    246      """
    247
    248      environment = {}
    249      for key in html.__all__:
    250          environment[key] = getattr(html, key)
    251      for key in validators.__all__:
    252          environment[key] = getattr(validators, key)
    253      if not request.env:
    254          request.env = Storage()
    255
    256      t = environment['T'] = translator(request)
    257      c = environment['cache'] = Cache(request)
    258      if store_current:
    259          current.request = request
    260          current.response = response
    261          current.session = session
    262          current.T = t
    263          current.cache = c
    264
    265      global __builtins__
    266      if is_jython: # jython hack
    267          __builtins__ = mybuiltin()
    268      else:
    269          __builtins__['__import__'] = __builtin__.__import__
    270      environment['__builtins__'] = __builtins__
    271      environment['HTTP'] = HTTP
    272      environment['redirect'] = redirect
    273      environment['request'] = request
    274      environment['response'] = response
    275      environment['session'] = session
    276      environment['DAL'] = DAL
    277      environment['Field'] = Field
    278      environment['SQLDB'] = SQLDB        # for backward compatibility
    279      environment['SQLField'] = SQLField  # for backward compatibility
    280      environment['SQLFORM'] = SQLFORM
    281      environment['SQLTABLE'] = SQLTABLE
    282      environment['LOAD'] = LoadFactory(environment)
    283      environment['local_import'] = \
    284          lambda name, reload=False, app=request.application:\
    285          local_import_aux(name,reload,app)
    286      BaseAdapter.set_folder(os.path.join(request.folder, 'databases'))
    287      response._view_environment = copy.copy(environment)
    288      return environment
    289
    290
    291 -def save_pyc(filename):
    292      """
    293      Bytecode compiles the file `filename`
    294      """
    295      py_compile.compile(filename)
    296
    297
    298 -def read_pyc(filename):
    299      """
    300      Read the code inside a bytecode compiled file if the MAGIC number is
    301      compatible
    302
    303      :returns: a code object
    304      """
    305      data = read_file(filename, 'rb')
    306      if not is_gae and data[:4] != imp.get_magic():
    307          raise SystemError, 'compiled code is incompatible'
    308      return marshal.loads(data[8:])
    309
    310
    311 -def compile_views(folder):
    312      """
    313      Compiles all the views in the application specified by `folder`
    314      """
    315
    316      path = os.path.join(folder, 'views')
    317      for file in listdir(path, '^[\w/]+\.\w+$'):
    318          data = parse_template(file, path)
    319          filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_')
    320          filename = os.path.join(folder, 'compiled', filename)
    321          write_file(filename, data)
    322          save_pyc(filename)
    323          os.unlink(filename)
    324
    325
    326 -def compile_models(folder):
    327      """
    328      Compiles all the models in the application specified by `folder`
    329      """
    330
    331      path = os.path.join(folder, 'models')
    332      for file in listdir(path, '.+\.py$'):
    333          data = read_file(os.path.join(path, file))
    334          filename = os.path.join(folder, 'compiled','models',file)
    335          mktree(filename)
    336          write_file(filename, data)
    337          save_pyc(filename)
    338          os.unlink(filename)
    339
    340
    341 -def compile_controllers(folder):
    342      """
    343      Compiles all the controllers in the application specified by `folder`
    344      """
    345
    346      path = os.path.join(folder, 'controllers')
    347      for file in listdir(path, '.+\.py$'):
    348          ### why is this here? save_pyc(os.path.join(path, file))
    349          data = read_file(os.path.join(path,file))
    350          exposed = regex_expose.findall(data)
    351          for function in exposed:
    352              command = data + "\nresponse._vars=response._caller(%s)\n" % \
    353                  function
    354              filename = os.path.join(folder, 'compiled', ('controllers/'
    355                                       + file[:-3]).replace('/', '_')
    356                                       + '_' + function + '.py')
    357              write_file(filename, command)
    358              save_pyc(filename)
    359              os.unlink(filename)
    360
    361
    362 -def run_models_in(environment):
    363      """
    364      Runs all models (in the app specified by the current folder)
    365      It tries pre-compiled models first before compiling them.
    366      """
    367
    368      folder = environment['request'].folder
    369      c = environment['request'].controller
    370      f = environment['request'].function
    371      cpath = os.path.join(folder, 'compiled')
    372      if os.path.exists(cpath):
    373          for model in listdir(cpath, '^models_\w+\.pyc$', 0):
    374              restricted(read_pyc(model), environment, layer=model)
    375          path = os.path.join(cpath, 'models')
    376          models = listdir(path, '^\w+\.pyc$',0,sort=False)
    377          compiled=True
    378      else:
    379          path = os.path.join(folder, 'models')
    380          models = listdir(path, '^\w+\.py$',0,sort=False)
    381          compiled=False
    382      paths = (path, os.path.join(path,c), os.path.join(path,c,f))
    383      for model in models:
    384          if not os.path.split(model)[0] in paths and c!='appadmin':
    385              continue
    386          elif compiled:
    387              code = read_pyc(model)
    388          elif is_gae:
    389              code = getcfs(model, model,
    390                            lambda: compile2(read_file(model), model))
    391          else:
    392              code = getcfs(model, model, None)
    393          restricted(code, environment, layer=model)
    394
    395
    396 -def run_controller_in(controller, function, environment):
    397      """
    398      Runs the controller.function() (for the app specified by
    399      the current folder).
    400      It tries pre-compiled controller_function.pyc first before compiling it.
    401      """
    402
    403      # if compiled should run compiled!
    404
    405      folder = environment['request'].folder
    406      path = os.path.join(folder, 'compiled')
    407      badc = 'invalid controller (%s/%s)' % (controller, function)
    408      badf = 'invalid function (%s/%s)' % (controller, function)
    409      if os.path.exists(path):
    410          filename = os.path.join(path, 'controllers_%s_%s.pyc'
    411                                   % (controller, function))
    412          if not os.path.exists(filename):
    413              raise HTTP(404,
    414                         rewrite.thread.routes.error_message % badf,
    415                         web2py_error=badf)
    416          restricted(read_pyc(filename), environment, layer=filename)
    417      elif function == '_TEST':
    418          # TESTING: adjust the path to include site packages
    419          from settings import global_settings
    420          from admin import abspath, add_path_first
    421          paths = (global_settings.gluon_parent, abspath('site-packages', gluon=True),  abspath('gluon', gluon=True), '')
    422          [add_path_first(path) for path in paths]
    423          # TESTING END
    424
    425          filename = os.path.join(folder, 'controllers/%s.py'
    426                                   % controller)
    427          if not os.path.exists(filename):
    428              raise HTTP(404,
    429                         rewrite.thread.routes.error_message % badc,
    430                         web2py_error=badc)
    431          environment['__symbols__'] = environment.keys()
    432          code = read_file(filename)
    433          code += TEST_CODE
    434          restricted(code, environment, layer=filename)
    435      else:
    436          filename = os.path.join(folder, 'controllers/%s.py'
    437                                   % controller)
    438          if not os.path.exists(filename):
    439              raise HTTP(404,
    440                         rewrite.thread.routes.error_message % badc,
    441                         web2py_error=badc)
    442          code = read_file(filename)
    443          exposed = regex_expose.findall(code)
    444          if not function in exposed:
    445              raise HTTP(404,
    446                         rewrite.thread.routes.error_message % badf,
    447                         web2py_error=badf)
    448          code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function)
    449          if is_gae:
    450              layer = filename + ':' + function
    451              code = getcfs(layer, filename,  lambda: compile2(code,layer))
    452          restricted(code, environment, filename)
    453      response = environment['response']
    454      vars=response._vars
    455      if response.postprocessing:
    456          for p in response.postprocessing:
    457              vars = p(vars)
    458      if isinstance(vars,unicode):
    459          vars = vars.encode('utf8')
    460      if hasattr(vars,'xml'):
    461          vars = vars.xml()
    462      return vars
    463
    464 -def run_view_in(environment):
    465      """
    466      Executes the view for the requested action.
    467      The view is the one specified in `response.view` or determined by the url
    468      or `view/generic.extension`
    469      It tries the pre-compiled views_controller_function.pyc before compiling it.
    470      """
    471
    472      request = environment['request']
    473      response = environment['response']
    474      folder = request.folder
    475      path = os.path.join(folder, 'compiled')
    476      badv = 'invalid view (%s)' % response.view
    477      patterns = response.generic_patterns or []
    478      regex = re.compile('|'.join(fnmatch.translate(r) for r in patterns))
    479      short_action =  '%(controller)s/%(function)s.%(extension)s' % request
    480      allow_generic = patterns and regex.search(short_action)
    481      if not isinstance(response.view, str):
    482          ccode = parse_template(response.view, os.path.join(folder, 'views'),
    483                                 context=environment)
    484          restricted(ccode, environment, 'file stream')
    485      elif os.path.exists(path):
    486          x = response.view.replace('/', '_')
    487          files = ['views_%s.pyc' % x]
    488          if allow_generic:
    489              files.append('views_generic.%s.pyc' % request.extension)
    490          # for backward compatibility
    491          if request.extension == 'html':
    492              files.append('views_%s.pyc' % x[:-5])
    493              if allow_generic:
    494                  files.append('views_generic.pyc')
    495          # end backward compatibility code
    496          for f in files:
    497              filename = os.path.join(path,f)
    498              if os.path.exists(filename):
    499                  code = read_pyc(filename)
    500                  restricted(code, environment, layer=filename)
    501                  return
    502          raise HTTP(404,
    503                     rewrite.thread.routes.error_message % badv,
    504                     web2py_error=badv)
    505      else:
    506          filename = os.path.join(folder, 'views', response.view)
    507          if not os.path.exists(filename) and allow_generic:
    508              response.view = 'generic.' + request.extension
    509              filename = os.path.join(folder, 'views', response.view)
    510          if not os.path.exists(filename):
    511              raise HTTP(404,
    512                         rewrite.thread.routes.error_message % badv,
    513                         web2py_error=badv)
    514          layer = filename
    515          if is_gae:
    516              ccode = getcfs(layer, filename,
    517                             lambda: compile2(parse_template(response.view,
    518                                              os.path.join(folder, 'views'),
    519                                              context=environment),layer))
    520          else:
    521              ccode = parse_template(response.view,
    522                                     os.path.join(folder, 'views'),
    523                                     context=environment)
    524          restricted(ccode, environment, layer)
    525
    526 -def remove_compiled_application(folder):
    527      """
    528      Deletes the folder `compiled` containing the compiled application.
    529      """
    530      try:
    531          shutil.rmtree(os.path.join(folder, 'compiled'))
    532          path = os.path.join(folder, 'controllers')
    533          for file in listdir(path,'.*\.pyc$',drop=False):
    534              os.unlink(file)
    535      except OSError:
    536          pass
    537
    538
    539 -def compile_application(folder):
    540      """
    541      Compiles all models, views, controller for the application in `folder`.
    542      """
    543      remove_compiled_application(folder)
    544      os.mkdir(os.path.join(folder, 'compiled'))
    545      compile_models(folder)
    546      compile_controllers(folder)
    547      compile_views(folder)
    548
    549
    550 -def test():
    551      """
    552      Example::
    553
    554          >>> import traceback, types
    555          >>> environment={'x':1}
    556          >>> open('a.py', 'w').write('print 1/x')
    557          >>> save_pyc('a.py')
    558          >>> os.unlink('a.py')
    559          >>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code'
    560          code
    561          >>> exec read_pyc('a.pyc') in environment
    562          1
    563      """
    564
    565      return
    566
    567
    568  if __name__ == '__main__':
    569      import doctest
    570      doctest.testmod()
    571