Package web2py :: Package gluon :: Module newcron |
[hide private]
[frames] | no frames]

# Source Code for Module web2py.gluon.newcron

      1  #!/usr/bin/env python
      2  # -*- coding: utf-8 -*-
      3
      4  """
      5  Created by Attila Csipa <web2py@csipa.in.rs>
      6  Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu>
      7  """
      8
      9  import sys
     10  import os
     11  import threading
     12  import logging
     13  import time
     14  import sched
     15  import re
     16  import datetime
     17  import platform
     18  import portalocker
     19  import fileutils
     20  import cPickle
     21  from settings import global_settings
     22
     23  logger = logging.getLogger("web2py.cron")
     24  _cron_stopping = False
     25
     26 -def stopcron():
     27      "graceful shutdown of cron"
     28      global _cron_stopping
     29      _cron_stopping = True
     30
     31 -class extcron(threading.Thread):
     32
     33 -    def __init__(self, applications_parent):
     34          threading.Thread.__init__(self)
     35          self.setDaemon(False)
     36          self.path = applications_parent
     37          crondance(self.path, 'external', startup=True)
     38
     39 -    def run(self):
     40          if not _cron_stopping:
     41              logger.debug('external cron invocation')
     42              crondance(self.path, 'external', startup=False)
     43
     44 -class hardcron(threading.Thread):
     45
     46 -    def __init__(self, applications_parent):
     47          threading.Thread.__init__(self)
     48          self.setDaemon(True)
     49          self.path = applications_parent
     50          crondance(self.path, 'hard', startup=True)
     51
     52 -    def launch(self):
     53          if not _cron_stopping:
     54              logger.debug('hard cron invocation')
     55              crondance(self.path, 'hard', startup = False)
     56
     57 -    def run(self):
     58          s = sched.scheduler(time.time, time.sleep)
     59          logger.info('Hard cron daemon started')
     60          while not _cron_stopping:
     61              now = time.time()
     62              s.enter(60 - now % 60, 1, self.launch, ())
     63              s.run()
     64
     65 -class softcron(threading.Thread):
     66
     67 -    def __init__(self, applications_parent):
     68          threading.Thread.__init__(self)
     69          self.path = applications_parent
     70          crondance(self.path, 'soft', startup=True)
     71
     72 -    def run(self):
     73          if not _cron_stopping:
     74              logger.debug('soft cron invocation')
     75              crondance(self.path, 'soft', startup=False)
     76
     77 -class Token(object):
     78
     79 -    def __init__(self,path):
     80          self.path = os.path.join(path, 'cron.master')
     81          if not os.path.exists(self.path):
     82              fileutils.write_file(self.path, '', 'wb')
     83          self.master = None
     84          self.now = time.time()
     85
     86 -    def acquire(self,startup=False):
     87          """
     88          returns the time when the lock is acquired or
     89          None if cron already running
     90
     91          lock is implemented by writing a pickle (start, stop) in cron.master
     92          start is time when cron job starts and stop is time when cron completed
     93          stop == 0 if job started but did not yet complete
     94          if a cron job started within less than 60 seconds, acquire returns None
     95          if a cron job started before 60 seconds and did not stop,
     96          a warning is issue "Stale cron.master detected"
     97          """
     98          if portalocker.LOCK_EX == None:
     99              logger.warning('WEB2PY CRON: Disabled because no file locking')
    100              return None
    101          self.master = open(self.path,'rb+')
    102          try:
    103              ret = None
    104              portalocker.lock(self.master,portalocker.LOCK_EX)
    105              try:
    106                  (start, stop) =  cPickle.load(self.master)
    107              except:
    108                  (start, stop) = (0, 1)
    109              if startup or self.now - start > 59.99:
    110                  ret = self.now
    111                  if not stop:
    112                      # this happens if previous cron job longer than 1 minute
    113                      logger.warning('WEB2PY CRON: Stale cron.master detected')
    114                  logger.debug('WEB2PY CRON: Acquiring lock')
    115                  self.master.seek(0)
    116                  cPickle.dump((self.now,0),self.master)
    117          finally:
    118              portalocker.unlock(self.master)
    119          if not ret:
    120              # do this so no need to release
    121              self.master.close()
    122          return ret
    123
    124 -    def release(self):
    125          """
    126          this function writes into cron.master the time when cron job
    127          was completed
    128          """
    129          if not self.master.closed:
    130              portalocker.lock(self.master,portalocker.LOCK_EX)
    131              logger.debug('WEB2PY CRON: Releasing cron lock')
    132              self.master.seek(0)
    133              (start, stop) =  cPickle.load(self.master)
    134              if start == self.now: # if this is my lock
    135                  self.master.seek(0)
    136                  cPickle.dump((self.now,time.time()),self.master)
    137              portalocker.unlock(self.master)
    138              self.master.close()
    139
    140
    141 -def rangetolist(s, period='min'):
    142      retval = []
    143      if s.startswith('*'):
    144          if period == 'min':
    145              s = s.replace('*', '0-59', 1)
    146          elif period == 'hr':
    147              s = s.replace('*', '0-23', 1)
    148          elif period == 'dom':
    149              s = s.replace('*', '1-31', 1)
    150          elif period == 'mon':
    151              s = s.replace('*', '1-12', 1)
    152          elif period == 'dow':
    153              s = s.replace('*', '0-6', 1)
    154      m = re.compile(r'(\d+)-(\d+)/(\d+)')
    155      match = m.match(s)
    156      if match:
    157          for i in range(int(match.group(1)), int(match.group(2)) + 1):
    158              if i % int(match.group(3)) == 0:
    159                  retval.append(i)
    160      return retval
    161
    162
    163 -def parsecronline(line):
    164      task = {}
    165      if line.startswith('@reboot'):
    166          line=line.replace('@reboot', '-1 * * * *')
    167      elif line.startswith('@yearly'):
    168          line=line.replace('@yearly', '0 0 1 1 *')
    169      elif line.startswith('@annually'):
    170          line=line.replace('@annually', '0 0 1 1 *')
    171      elif line.startswith('@monthly'):
    172          line=line.replace('@monthly', '0 0 1 * *')
    173      elif line.startswith('@weekly'):
    174          line=line.replace('@weekly', '0 0 * * 0')
    175      elif line.startswith('@daily'):
    176          line=line.replace('@daily', '0 0 * * *')
    177      elif line.startswith('@midnight'):
    178          line=line.replace('@midnight', '0 0 * * *')
    179      elif line.startswith('@hourly'):
    180          line=line.replace('@hourly', '0 * * * *')
    181      params = line.strip().split(None, 6)
    182      if len(params) < 7:
    183          return None
    184      daysofweek={'sun':0,'mon':1,'tue':2,'wed':3,'thu':4,'fri':5,'sat':6}
    185      for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']):
    186          if not s in [None, '*']:
    187              task[id] = []
    188              vals = s.split(',')
    189              for val in vals:
    190                  if val != '-1' and '-' in val and '/' not in val:
    191                      val = '%s/1' % val
    192                  if '/' in val:
    193                      task[id] += rangetolist(val, id)
    194                  elif val.isdigit() or val=='-1':
    195                      task[id].append(int(val))
    196                  elif id=='dow' and val[:3].lower() in daysofweek:
    197                      task[id].append(daysofweek(val[:3].lower()))
    198      task['user'] = params[5]
    199      task['cmd'] = params[6]
    200      return task
    201
    202
    203 -class cronlauncher(threading.Thread):
    204
    205 -    def __init__(self, cmd, shell=True):
    206          threading.Thread.__init__(self)
    207          if platform.system() == 'Windows':
    208              shell = False
    209          elif isinstance(cmd,list):
    210              cmd = ' '.join(cmd)
    211          self.cmd = cmd
    212          self.shell = shell
    213
    214 -    def run(self):
    215          import subprocess
    216          proc = subprocess.Popen(self.cmd,
    217                                  stdin=subprocess.PIPE,
    218                                  stdout=subprocess.PIPE,
    219                                  stderr=subprocess.PIPE,
    220                                  shell=self.shell)
    221          (stdoutdata,stderrdata) = proc.communicate()
    222          if proc.returncode != 0:
    223              logger.warning(
    224                  'WEB2PY CRON Call returned code %s:\n%s' % \
    225                      (proc.returncode, stdoutdata+stderrdata))
    226          else:
    227              logger.debug('WEB2PY CRON Call returned success:\n%s' \
    228                                % stdoutdata)
    229
    230 -def crondance(applications_parent, ctype='soft', startup=False):
    231      apppath = os.path.join(applications_parent,'applications')
    232      cron_path = os.path.join(apppath,'admin','cron')
    233      token = Token(cron_path)
    234      cronmaster = token.acquire(startup=startup)
    235      if not cronmaster:
    236          return
    237      now_s = time.localtime()
    238      checks=(('min',now_s.tm_min),
    239              ('hr',now_s.tm_hour),
    240              ('mon',now_s.tm_mon),
    241              ('dom',now_s.tm_mday),
    242              ('dow',(now_s.tm_wday+1)%7))
    243
    244      apps = [x for x in os.listdir(apppath)
    245              if os.path.isdir(os.path.join(apppath, x))]
    246
    247      for app in apps:
    248          if _cron_stopping:
    249              break;
    250          apath = os.path.join(apppath,app)
    251          cronpath = os.path.join(apath, 'cron')
    252          crontab = os.path.join(cronpath, 'crontab')
    253          if not os.path.exists(crontab):
    254              continue
    255          try:
    256              cronlines = fileutils.readlines_file(crontab, 'rt')
    257              lines = [x.strip() for x in cronlines if x.strip() and not x.strip().startswith('#')]
    258              tasks = [parsecronline(cline) for cline in lines]
    259          except Exception, e:
    260              logger.error('WEB2PY CRON: crontab read error %s' % e)
    261              continue
    262
    263          for task in tasks:
    264              if _cron_stopping:
    265                  break;
    266              commands = [sys.executable]
    267              w2p_path = fileutils.abspath('web2py.py', gluon=True)
    268              if os.path.exists(w2p_path):
    269                  commands.append(w2p_path)
    270              if global_settings.applications_parent != global_settings.gluon_parent:
    271                  commands.extend(('-f', global_settings.applications_parent))
    272              citems = [(k in task and not v in task[k]) for k,v in checks]
    273              task_min= task.get('min',[])
    274              if not task:
    275                  continue
    276              elif not startup and task_min == [-1]:
    277                  continue
    278              elif task_min != [-1] and reduce(lambda a,b: a or b, citems):
    279                  continue
    280              logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' \
    281                               % (ctype, app, task.get('cmd'),
    282                                  os.getcwd(), datetime.datetime.now()))
    283              action, command, models = False, task['cmd'], ''
    284              if command.startswith('**'):
    285                  (action,models,command) = (True,'',command[2:])
    286              elif command.startswith('*'):
    287                  (action,models,command) = (True,'-M',command[1:])
    288              else:
    289                  action=False
    290              if action and command.endswith('.py'):
    291                  commands.extend(('-J',                # cron job
    292                                   models,              # import models?
    293                                   '-S', app,           # app name
    294                                   '-a', '"<recycle>"', # password
    295                                   '-R', command))      # command
    296                  shell = True
    297              elif action:
    298                  commands.extend(('-J',                  # cron job
    299                                   models,                # import models?
    300                                   '-S', app+'/'+command, # app name
    301                                   '-a', '"<recycle>"'))  # password
    302                  shell = True
    303              else:
    304                  commands = command
    305                  shell = False
    306              try:
    307                  cronlauncher(commands, shell=shell).start()
    308              except Exception, e:
    309                  logger.warning(
    310                      'WEB2PY CRON: Execution error for %s: %s' \
    311                          % (task.get('cmd'), e))
    312      token.release()
    313