Package web2py :: Package gluon :: Module newcron
[hide private]
[frames] | no frames]

Source Code for Module web2py.gluon.newcron

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  """ 
  5  Created by Attila Csipa <web2py@csipa.in.rs> 
  6  Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
  7  """ 
  8   
  9  import sys 
 10  import os 
 11  import threading 
 12  import logging 
 13  import time 
 14  import sched 
 15  import re 
 16  import datetime 
 17  import platform 
 18  import portalocker 
 19  import fileutils 
 20  import cPickle 
 21  from settings import global_settings 
 22   
 23  logger = logging.getLogger("web2py.cron") 
 24  _cron_stopping = False 
 25   
26 -def stopcron():
27 "graceful shutdown of cron" 28 global _cron_stopping 29 _cron_stopping = True
30
31 -class extcron(threading.Thread):
32
33 - def __init__(self, applications_parent):
34 threading.Thread.__init__(self) 35 self.setDaemon(False) 36 self.path = applications_parent 37 crondance(self.path, 'external', startup=True)
38
39 - def run(self):
40 if not _cron_stopping: 41 logger.debug('external cron invocation') 42 crondance(self.path, 'external', startup=False)
43
44 -class hardcron(threading.Thread):
45
46 - def __init__(self, applications_parent):
47 threading.Thread.__init__(self) 48 self.setDaemon(True) 49 self.path = applications_parent 50 crondance(self.path, 'hard', startup=True)
51
52 - def launch(self):
53 if not _cron_stopping: 54 logger.debug('hard cron invocation') 55 crondance(self.path, 'hard', startup = False)
56
57 - def run(self):
58 s = sched.scheduler(time.time, time.sleep) 59 logger.info('Hard cron daemon started') 60 while not _cron_stopping: 61 now = time.time() 62 s.enter(60 - now % 60, 1, self.launch, ()) 63 s.run()
64
65 -class softcron(threading.Thread):
66
67 - def __init__(self, applications_parent):
68 threading.Thread.__init__(self) 69 self.path = applications_parent 70 crondance(self.path, 'soft', startup=True)
71
72 - def run(self):
73 if not _cron_stopping: 74 logger.debug('soft cron invocation') 75 crondance(self.path, 'soft', startup=False)
76
77 -class Token(object):
78
79 - def __init__(self,path):
80 self.path = os.path.join(path, 'cron.master') 81 if not os.path.exists(self.path): 82 fileutils.write_file(self.path, '', 'wb') 83 self.master = None 84 self.now = time.time()
85
86 - def acquire(self,startup=False):
87 """ 88 returns the time when the lock is acquired or 89 None if cron already running 90 91 lock is implemented by writing a pickle (start, stop) in cron.master 92 start is time when cron job starts and stop is time when cron completed 93 stop == 0 if job started but did not yet complete 94 if a cron job started within less than 60 seconds, acquire returns None 95 if a cron job started before 60 seconds and did not stop, 96 a warning is issue "Stale cron.master detected" 97 """ 98 if portalocker.LOCK_EX == None: 99 logger.warning('WEB2PY CRON: Disabled because no file locking') 100 return None 101 self.master = open(self.path,'rb+') 102 try: 103 ret = None 104 portalocker.lock(self.master,portalocker.LOCK_EX) 105 try: 106 (start, stop) = cPickle.load(self.master) 107 except: 108 (start, stop) = (0, 1) 109 if startup or self.now - start > 59.99: 110 ret = self.now 111 if not stop: 112 # this happens if previous cron job longer than 1 minute 113 logger.warning('WEB2PY CRON: Stale cron.master detected') 114 logger.debug('WEB2PY CRON: Acquiring lock') 115 self.master.seek(0) 116 cPickle.dump((self.now,0),self.master) 117 finally: 118 portalocker.unlock(self.master) 119 if not ret: 120 # do this so no need to release 121 self.master.close() 122 return ret
123
124 - def release(self):
125 """ 126 this function writes into cron.master the time when cron job 127 was completed 128 """ 129 if not self.master.closed: 130 portalocker.lock(self.master,portalocker.LOCK_EX) 131 logger.debug('WEB2PY CRON: Releasing cron lock') 132 self.master.seek(0) 133 (start, stop) = cPickle.load(self.master) 134 if start == self.now: # if this is my lock 135 self.master.seek(0) 136 cPickle.dump((self.now,time.time()),self.master) 137 portalocker.unlock(self.master) 138 self.master.close()
139 140
141 -def rangetolist(s, period='min'):
142 retval = [] 143 if s.startswith('*'): 144 if period == 'min': 145 s = s.replace('*', '0-59', 1) 146 elif period == 'hr': 147 s = s.replace('*', '0-23', 1) 148 elif period == 'dom': 149 s = s.replace('*', '1-31', 1) 150 elif period == 'mon': 151 s = s.replace('*', '1-12', 1) 152 elif period == 'dow': 153 s = s.replace('*', '0-6', 1) 154 m = re.compile(r'(\d+)-(\d+)/(\d+)') 155 match = m.match(s) 156 if match: 157 for i in range(int(match.group(1)), int(match.group(2)) + 1): 158 if i % int(match.group(3)) == 0: 159 retval.append(i) 160 return retval
161 162
163 -def parsecronline(line):
164 task = {} 165 if line.startswith('@reboot'): 166 line=line.replace('@reboot', '-1 * * * *') 167 elif line.startswith('@yearly'): 168 line=line.replace('@yearly', '0 0 1 1 *') 169 elif line.startswith('@annually'): 170 line=line.replace('@annually', '0 0 1 1 *') 171 elif line.startswith('@monthly'): 172 line=line.replace('@monthly', '0 0 1 * *') 173 elif line.startswith('@weekly'): 174 line=line.replace('@weekly', '0 0 * * 0') 175 elif line.startswith('@daily'): 176 line=line.replace('@daily', '0 0 * * *') 177 elif line.startswith('@midnight'): 178 line=line.replace('@midnight', '0 0 * * *') 179 elif line.startswith('@hourly'): 180 line=line.replace('@hourly', '0 * * * *') 181 params = line.strip().split(None, 6) 182 if len(params) < 7: 183 return None 184 daysofweek={'sun':0,'mon':1,'tue':2,'wed':3,'thu':4,'fri':5,'sat':6} 185 for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): 186 if not s in [None, '*']: 187 task[id] = [] 188 vals = s.split(',') 189 for val in vals: 190 if val != '-1' and '-' in val and '/' not in val: 191 val = '%s/1' % val 192 if '/' in val: 193 task[id] += rangetolist(val, id) 194 elif val.isdigit() or val=='-1': 195 task[id].append(int(val)) 196 elif id=='dow' and val[:3].lower() in daysofweek: 197 task[id].append(daysofweek(val[:3].lower())) 198 task['user'] = params[5] 199 task['cmd'] = params[6] 200 return task
201 202
203 -class cronlauncher(threading.Thread):
204
205 - def __init__(self, cmd, shell=True):
206 threading.Thread.__init__(self) 207 if platform.system() == 'Windows': 208 shell = False 209 elif isinstance(cmd,list): 210 cmd = ' '.join(cmd) 211 self.cmd = cmd 212 self.shell = shell
213
214 - def run(self):
215 import subprocess 216 proc = subprocess.Popen(self.cmd, 217 stdin=subprocess.PIPE, 218 stdout=subprocess.PIPE, 219 stderr=subprocess.PIPE, 220 shell=self.shell) 221 (stdoutdata,stderrdata) = proc.communicate() 222 if proc.returncode != 0: 223 logger.warning( 224 'WEB2PY CRON Call returned code %s:\n%s' % \ 225 (proc.returncode, stdoutdata+stderrdata)) 226 else: 227 logger.debug('WEB2PY CRON Call returned success:\n%s' \ 228 % stdoutdata)
229
230 -def crondance(applications_parent, ctype='soft', startup=False):
231 apppath = os.path.join(applications_parent,'applications') 232 cron_path = os.path.join(apppath,'admin','cron') 233 token = Token(cron_path) 234 cronmaster = token.acquire(startup=startup) 235 if not cronmaster: 236 return 237 now_s = time.localtime() 238 checks=(('min',now_s.tm_min), 239 ('hr',now_s.tm_hour), 240 ('mon',now_s.tm_mon), 241 ('dom',now_s.tm_mday), 242 ('dow',(now_s.tm_wday+1)%7)) 243 244 apps = [x for x in os.listdir(apppath) 245 if os.path.isdir(os.path.join(apppath, x))] 246 247 for app in apps: 248 if _cron_stopping: 249 break; 250 apath = os.path.join(apppath,app) 251 cronpath = os.path.join(apath, 'cron') 252 crontab = os.path.join(cronpath, 'crontab') 253 if not os.path.exists(crontab): 254 continue 255 try: 256 cronlines = fileutils.readlines_file(crontab, 'rt') 257 lines = [x.strip() for x in cronlines if x.strip() and not x.strip().startswith('#')] 258 tasks = [parsecronline(cline) for cline in lines] 259 except Exception, e: 260 logger.error('WEB2PY CRON: crontab read error %s' % e) 261 continue 262 263 for task in tasks: 264 if _cron_stopping: 265 break; 266 commands = [sys.executable] 267 w2p_path = fileutils.abspath('web2py.py', gluon=True) 268 if os.path.exists(w2p_path): 269 commands.append(w2p_path) 270 if global_settings.applications_parent != global_settings.gluon_parent: 271 commands.extend(('-f', global_settings.applications_parent)) 272 citems = [(k in task and not v in task[k]) for k,v in checks] 273 task_min= task.get('min',[]) 274 if not task: 275 continue 276 elif not startup and task_min == [-1]: 277 continue 278 elif task_min != [-1] and reduce(lambda a,b: a or b, citems): 279 continue 280 logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' \ 281 % (ctype, app, task.get('cmd'), 282 os.getcwd(), datetime.datetime.now())) 283 action, command, models = False, task['cmd'], '' 284 if command.startswith('**'): 285 (action,models,command) = (True,'',command[2:]) 286 elif command.startswith('*'): 287 (action,models,command) = (True,'-M',command[1:]) 288 else: 289 action=False 290 if action and command.endswith('.py'): 291 commands.extend(('-J', # cron job 292 models, # import models? 293 '-S', app, # app name 294 '-a', '"<recycle>"', # password 295 '-R', command)) # command 296 shell = True 297 elif action: 298 commands.extend(('-J', # cron job 299 models, # import models? 300 '-S', app+'/'+command, # app name 301 '-a', '"<recycle>"')) # password 302 shell = True 303 else: 304 commands = command 305 shell = False 306 try: 307 cronlauncher(commands, shell=shell).start() 308 except Exception, e: 309 logger.warning( 310 'WEB2PY CRON: Execution error for %s: %s' \ 311 % (task.get('cmd'), e)) 312 token.release()
313