Random Bits of Open Code

Check-in [30dd9a8f0e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Added function to make a fake db
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:30dd9a8f0e641fcddae4e9aad57a5ed86c5a58ef
User & Date: matt 2019-03-26 04:59:43
Context
2019-03-26
05:34
yada check-in: 460885298e user: matt tags: trunk
04:59
Added function to make a fake db check-in: 30dd9a8f0e user: matt tags: trunk
04:25
cleanup check-in: 9057c1187c user: matt tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ringchart/process-nbjobs.scm.

19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35



36
37
38
39
40
41
42
43
44











45
46
47
48
49
50
51
52






























53
54
55
56
57
58
59
...
180
181
182
183
184
185
186









187





   ((string? num) (string->number num))
   (else num)))

;;======================================================================
;; J O B S 
;;======================================================================

(define (get-jobs)
  (for-each
   (lambda (fname)
     (let* ((zipped   (string-match (regexp ".*\\.gz$") fname)) ;; true is gzip, false is plain
	    (dbexists (file-exists? "output.db"))
	    (db       (open-database "output.db")))
       ;; (exec (sql db "PRAGMA synchronous=0;"))
       (execute db "PRAGMA synchronous=0;")
       (if (not dbexists)(init-db db))
       (print "Processing " fname)



       (let ((stmt      (prepare db "INSERT INTO jobs (jobid,iteration,user,submit_time,queue_name,num_cpus,mem,duration,host,start_time,finish_time,exit_status,cores_consumed) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?);"))
	     ;;                                         0       1       2        3          4        5      6     7       8       9          10         11           12
	     (dups-stmt (prepare db "SELECT jobid,iteration,user,submit_time,queue_name,num_cpus,mem,duration,host,start_time,finish_time,exit_status,cores_consumed FROM jobs
                                        WHERE jobid=? AND
                                              user=? AND
                                              host=? AND
                                              start_time=? AND
                                              submit_time=?;"))
	     (del-stmt  (prepare db "DELETE FROM jobs WHERE id=?;"))











	     (allids (get-all-job-ids db (make-hash-table))))
         ;;(print allids)
         ;;(format #t "~A~%" (hash-table-keys allids))
	 ((if zipped with-input-from-pipe with-input-from-file)
	  (if zipped (conc "zcat " fname) fname)
	   (lambda ()
	     (process-file db del-stmt dups-stmt stmt allids))))))
   (filter file-exists? (command-line-arguments))))































(define (process-file db del-stmt dups-stmt stmt alljobids)
  (let ((start-time (current-seconds)))
    (with-transaction
     db
     (lambda ()
       (let loop ((inl (read-line))
................................................................................
                  host TEXT,
                  start_time INTEGER,
                  finish_time INTEGER,
                  exit_status INTEGER,
                  cores_consumed INTEGER);"
	 "CREATE INDEX jobs_index ON jobs (jobid);")))










(get-jobs)












|
<
<
<
|
|
<
|
|
<
>
>
>
|
|
|





|
>
>
>
>
>
>
>
>
>
>
>
|






|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
19
20
21
22
23
24
25
26



27
28

29
30

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
...
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
   ((string? num) (string->number num))
   (else num)))

;;======================================================================
;; J O B S 
;;======================================================================

(define (create-get-db fname)



  (let* ((dbexists (file-exists? "output.db"))
	 (db       (open-database "output.db")))

    (execute db "PRAGMA synchronous=0;")
    (if (not dbexists)(init-db db))

    db))

(define (make-statements db)
  `((stmt .  ,(prepare db "INSERT INTO jobs (jobid,iteration,user,submit_time,queue_name,num_cpus,mem,duration,host,start_time,finish_time,exit_status,cores_consumed) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?);"))
    ;;                                         0       1       2        3          4        5      6     7       8       9          10         11           12
    (dups-stmt . ,(prepare db "SELECT jobid,iteration,user,submit_time,queue_name,num_cpus,mem,duration,host,start_time,finish_time,exit_status,cores_consumed FROM jobs
                                        WHERE jobid=? AND
                                              user=? AND
                                              host=? AND
                                              start_time=? AND
                                              submit_time=?;"))
    (del-stmt  . ,(prepare db "DELETE FROM jobs WHERE id=?;"))))

(define (get-jobs)
  (for-each
   (lambda (fname)
     (let* ((zipped   (string-match (regexp ".*\\.gz$") fname)) ;; true is gzip, false is plain
	    (db       (create-get-db "output.db")))
       (print "Processing " fname)
       (let* ((stmts     (make-statements db))
	      (stmt      (alist-ref 'stmt stmts))
	      (dups-stmt (alist-ref 'dups-stmt stmts))
	      (del-stmt  (alist-ref 'del-stmt  stmts))
	      (allids    (get-all-job-ids db (make-hash-table))))
         ;;(print allids)
         ;;(format #t "~A~%" (hash-table-keys allids))
	 ((if zipped with-input-from-pipe with-input-from-file)
	  (if zipped (conc "zcat " fname) fname)
	   (lambda ()
	     (process-file db del-stmt dups-stmt stmt allids))))))
   (filter file-exists? (cdr (command-line-arguments))))) ;; first item on command line is the command

(define (make-fake-jobs)
  (let* ((db (create-get-db "output.db"))
	 (stmts (make-statements db))
	 (instmt (alist-ref 'stmt stmts))
	 (one-month  (* 30 24 3600))
	 (start-time (- (current-seconds) one-month)) ;; aprox one month ago
	 (end-time   (current-seconds)))
    (let loop ((jnum 0))
      (let* ((submit-time    (+ start-time -1000 (random one-month)))
	     (duration       (random 8000))
	     (job-start-time (+ submit-time (random 3600)))
	     (num-cores      (random 22)))
	(execute instmt
		 jnum ;; jobid
		 1    ;; iteration
		 (conc "User" (random 20))               ;; username
		 submit-time                             ;; submit_time
		 (conc "Queue" (random 5))               ;; queue name
		 num-cores                               ;; num cores
		 (random 1000)                           ;; memory
		 duration                                ;; duration
		 (conc "Host" (random 20))               ;; hostname
		 job-start-time                          ;; job start time
		 (+ job-start-time duration)             ;; job end time
		 0
		 (- num-cores (random num-cores))))      ;; num cores actually used
      (if (< jnum 10000)(loop (+ jnum 1))))))
	       


(define (process-file db del-stmt dups-stmt stmt alljobids)
  (let ((start-time (current-seconds)))
    (with-transaction
     db
     (lambda ()
       (let loop ((inl (read-line))
................................................................................
                  host TEXT,
                  start_time INTEGER,
                  finish_time INTEGER,
                  exit_status INTEGER,
                  cores_consumed INTEGER);"
	 "CREATE INDEX jobs_index ON jobs (jobid);")))

(define (help)
  (print "Usage: process-nbjobs <cmd> [file ...]")
  (print "   where cmd is one of:")
  (print "     fake-jobs   : make fake jobs database in output.db")
  (print "     get-jobs    : read files from commandline into output.db"))

(if (> (length (command-line-arguments)) 0)
    (case (string->symbol (car (command-line-arguments)))
      ((fake-jobs)(make-fake-jobs))
      ((process)  (get-jobs))
      (else (help)(exit 1)))
    (begin
      (help)
      (exit 1)))