Random Bits of Open Code

Check-in [4675203d76]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Added code for creating team based usage charts. Not working perfectly since large queries are killing sqlite
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:4675203d763cffb8f70a505027d79a49d73c94dd
User & Date: jmoon18 2019-04-29 16:23:42
Context
2019-05-01
17:20
Added detailed manager level graphs with sub-users, still missing sub-manager summaries check-in: e89d9090b5 user: jmoon18 tags: trunk
2019-04-29
16:23
Added code for creating team based usage charts. Not working perfectly since large queries are killing sqlite check-in: 4675203d76 user: jmoon18 tags: trunk
2019-04-26
23:04
Updated peak html generation for single users check-in: d0f85f520a user: jmoon18 tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ringchart/runstats.scm.

90
91
92
93
94
95
96
97
98
99
100
101
102
103
     (case (string->symbol cmd)
       ((all)
	(doit "people.db" id)
	(regen-rollup id)
        (print "ID: " id)
	(create-datafiles #f id)
	(close-database db))
       ((peak) (gen-peak-for-user param))
       ((rollup)(regen-rollup id))
       ((datafiles)(create-datafiles #f id)) 
       ((update)(doit "people.db" id))
       (else (print "Command " cmd " with param " param " not recognised")))
     )))
   







|






90
91
92
93
94
95
96
97
98
99
100
101
102
103
     (case (string->symbol cmd)
       ((all)
	(doit "people.db" id)
	(regen-rollup id)
        (print "ID: " id)
	(create-datafiles #f id)
	(close-database db))
       ((peak) (gen-peak-for-user param #t))
       ((rollup)(regen-rollup id))
       ((datafiles)(create-datafiles #f id)) 
       ((update)(doit "people.db" id))
       (else (print "Command " cmd " with param " param " not recognised")))
     )))
   

Changes to ringchart/src/cstats.scm.

6
7
8
9
10
11
12

13
14
15
16
17
18
19
..
21
22
23
24
25
26
27

28
29
30
31
32
33
34
...
472
473
474
475
476
477
478








479



480

481
482
483
484
485
486
487
488



489

490

491
492
493
494
495
496
497
...
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
...
630
631
632
633
634
635
636
637

638
639


640
641
642
643
644
645
646
647
648
649
650
651
 regex
 matchable
 miscmacros
 sql-de-lite
 sqlite3
 posix
 directory-utils

 srfi-69)

(module cstats
	*

(import
 scheme
................................................................................
 data-structures
 directory-utils
 regex
 matchable
 miscmacros
 ;; (prefix margs args:)
 sql-de-lite

 (prefix sqlite3 sql3:)
 posix
 chicken
 srfi-1
 srfi-69)

;;======================================================================
................................................................................
          alljobs
          )
          (print "Highest peak for " user " is " highest-peak)
        )
   )
)













(define (gen-peak-for-user user)

   (let* ((jobs-db  (open-db "jobs.db"))
          (old-max 0)
          (max-timestamp 0)
          (highest-peak 0)
          (highest-peak-cores 0)
          (last-peak 0)
          (last-write 0)
          (ht (make-hash-table))



          (sth (prepare jobs-db "SELECT * FROM (SELECT num_cpus,start_time AS blah, jobid FROM jobs WHERE user=? AND num_cpus > 0 UNION SELECT (0 - num_cpus) AS num_cpus,finish_time AS blah, jobid FROM jobs WHERE user=? and num_cpus > 0) ORDER BY blah ASC ;" )))

         (print "Getting peak data for " user)

         (query 
          (fold-rows
           ;;(lambda (res num_cpus event-time jobid)
           (lambda (my-row res)
             ;;(print my-row)
             (let ((newres (+ res (car my-row))))
               (if (> newres old-max) (set! old-max newres))
................................................................................
                 (begin (hash-table-set! ht (second my-row) newres) (set! last-write (second my-row)))
               )
               ;;(print jobid " " event-time " " num_cpus " " newres)
               (set! last-peak (car my-row))
               ;;(print (first my-row) "," newres)
               newres))
             0)
           sth (car user) (car user))
          (print "Max: " old-max)
          (list old-max max-timestamp ht)
))

(define (create-peak-html user peak-data)
  (print "Creating html for: " user)
  (if (not (directory? "peaks"))
................................................................................
  )
)

(define (gen-peak-data)
  (let* ((people-db (open-db "people.db")))
    (for-each
      (lambda(user)
        (let* ((peak-data (gen-peak-for-user user)))

          (create-peak-csv user peak-data)
          (create-peak-html user peak-data)


          (handle-exceptions exn (print "Update-peak-db failed" ((condition-property-accessor 'exn 'message) exn)) (update-peak-db  user peak-data))
          (print "Updated everything for " user)
        )
      )
    (query fetch-rows (sql people-db "select uid from people"))
    )
  ))




)







>







 







>







 







>
>
>
>
>
>
>
>
|
>
>
>
|
>








>
>
>
|
>

>







 







|







 







|
>

|
>
>












6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
..
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
...
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
...
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
...
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
 regex
 matchable
 miscmacros
 sql-de-lite
 sqlite3
 posix
 directory-utils
 srfi-13
 srfi-69)

(module cstats
	*

(import
 scheme
................................................................................
 data-structures
 directory-utils
 regex
 matchable
 miscmacros
 ;; (prefix margs args:)
 sql-de-lite
 srfi-13
 (prefix sqlite3 sql3:)
 posix
 chicken
 srfi-1
 srfi-69)

;;======================================================================
................................................................................
          alljobs
          )
          (print "Highest peak for " user " is " highest-peak)
        )
   )
)

(define (get-employees user)
  (let* ((db (open-db "people.db"))
         (sth (prepare db (conc "SELECT uid from people where mgr_id in (select id from people where uid='" user "' )")))
         (employees (fetch-rows sth)))
         ;;(list user (map get-employees (fetch-rows sth)))
         (if (>= (length+ employees) 1)
           (flatten (append (list user) (map get-employees (map car employees))))
           (list user))
  )
)


(define (gen-peak-for-user user manager)
   (print "User: " user " Manager: " manager)
   (let* ((jobs-db  (open-db "jobs.db"))
          (old-max 0)
          (max-timestamp 0)
          (highest-peak 0)
          (highest-peak-cores 0)
          (last-peak 0)
          (last-write 0)
          (ht (make-hash-table))
          (filter-users (if manager (get-employees (car user)) user))
          (foo (print "Filter-users: " filter-users))
          (filter-string (conc "('" (string-join filter-users "','")  "')"))
          (sql-query (conc "SELECT * FROM (SELECT num_cpus,start_time AS blah, jobid FROM jobs WHERE user in " filter-string " AND num_cpus > 0 UNION SELECT (0 - num_cpus) AS num_cpus,finish_time AS blah, jobid FROM jobs WHERE user in " filter-string " and num_cpus > 0) ORDER BY blah ASC ;"))
          (sth (prepare jobs-db sql-query)))
         (print "Getting peak data for " user)
         (print "Query: " sql-query)
         (query 
          (fold-rows
           ;;(lambda (res num_cpus event-time jobid)
           (lambda (my-row res)
             ;;(print my-row)
             (let ((newres (+ res (car my-row))))
               (if (> newres old-max) (set! old-max newres))
................................................................................
                 (begin (hash-table-set! ht (second my-row) newres) (set! last-write (second my-row)))
               )
               ;;(print jobid " " event-time " " num_cpus " " newres)
               (set! last-peak (car my-row))
               ;;(print (first my-row) "," newres)
               newres))
             0)
           sth)
          (print "Max: " old-max)
          (list old-max max-timestamp ht)
))

(define (create-peak-html user peak-data)
  (print "Creating html for: " user)
  (if (not (directory? "peaks"))
................................................................................
  )
)

(define (gen-peak-data)
  (let* ((people-db (open-db "people.db")))
    (for-each
      (lambda(user)
        (let* ((peak-data (gen-peak-for-user user #f))
               (peak-data-mgr (gen-peak-for-user user #t)))
          (create-peak-csv user peak-data)
          (create-peak-html  user peak-data)
          (create-peak-csv  (list (conc (car user) "-team")) peak-data-mgr)
          (create-peak-html (list (conc (car user) "-team")) peak-data-mgr)
          (handle-exceptions exn (print "Update-peak-db failed" ((condition-property-accessor 'exn 'message) exn)) (update-peak-db  user peak-data))
          (print "Updated everything for " user)
        )
      )
    (query fetch-rows (sql people-db "select uid from people"))
    )
  ))




)