Overview
Comment: | Better handling of stats -- keep them in a transient SQLite3 DB |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
a010d6c0a73c4ca7a221cf1a13f530b9 |
User & Date: | rkeene on 2019-01-12 21:48:59 |
Other Links: | manifest | tags |
Context
2019-01-12
| ||
21:59 | Fixed name of keepalive statistic for unique peers check-in: 173b00efe3 user: rkeene tags: trunk | |
21:48 | Better handling of stats -- keep them in a transient SQLite3 DB check-in: a010d6c0a7 user: rkeene tags: trunk | |
2019-01-11
| ||
18:03 | Reduce argon2 code a bit by removing features we do not use check-in: 483b914916 user: rkeene tags: trunk | |
Changes
Modified nano.tcl from [7e83f74db7] to [7eecb89b12].
︙ | ︙ | |||
20 21 22 23 24 25 26 27 28 29 30 31 32 33 | namespace eval ::nano::rpc {} namespace eval ::nano::rpc::client {} namespace eval ::nano::rpc::cli {} namespace eval ::nano::balance {} namespace eval ::nano::node::bootstrap {} namespace eval ::nano::node::realtime {} namespace eval ::nano::node::cli {} namespace eval ::nano::network::client {} namespace eval ::nano::network::server {} namespace eval ::nano::protocol::create {} namespace eval ::nano::protocol::parse {} namespace eval ::nano::protocol::extensions {} namespace eval ::nano::network::_dns {} namespace eval ::nano::wallet {} | > | 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | namespace eval ::nano::rpc {} namespace eval ::nano::rpc::client {} namespace eval ::nano::rpc::cli {} namespace eval ::nano::balance {} namespace eval ::nano::node::bootstrap {} namespace eval ::nano::node::realtime {} namespace eval ::nano::node::cli {} namespace eval ::nano::node::stats {} namespace eval ::nano::network::client {} namespace eval ::nano::network::server {} namespace eval ::nano::protocol::create {} namespace eval ::nano::protocol::parse {} namespace eval ::nano::protocol::extensions {} namespace eval ::nano::network::_dns {} namespace eval ::nano::wallet {} |
︙ | ︙ | |||
3336 3337 3338 3339 3340 3341 3342 | #puts "Querying $peerSock with node_id_handshake (1)" ::nano::network::client $peerSock "node_id_handshake" query -query $node_id_nonce } } } # Stats | | | | < | 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 | #puts "Querying $peerSock with node_id_handshake (1)" ::nano::network::client $peerSock "node_id_handshake" query -query $node_id_nonce } } } # Stats ::nano::node::stats::incr [list keepalive count] ::nano::node::stats::incr [list keepalive peers] [llength $peers] foreach peer $peers { ::nano::node::stats::lappend keepalive $peer } return "" } proc ::nano::protocol::parse::node_id_handshake {extensions messageData} { array set result [list] |
︙ | ︙ | |||
3372 3373 3374 3375 3376 3377 3378 | if {"query" in [dict get $messageDict flags]} { set query [dict get $messageDict query] set clientID [dict get $::nano::node::configuration node client_id_private_key] set retval [dict create "invoke_client" [list node_id_handshake response -privateKey $clientID -query [binary decode hex $query]]] # Stats | | | < | | | | | | < < | < < < | < < < < < < < < < < | | | | < < < | 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 | if {"query" in [dict get $messageDict flags]} { set query [dict get $messageDict query] set clientID [dict get $::nano::node::configuration node client_id_private_key] set retval [dict create "invoke_client" [list node_id_handshake response -privateKey $clientID -query [binary decode hex $query]]] # Stats ::nano::node::stats::incr [list node_id_handshake query count] } if {"response" in [dict get $messageDict flags]} { set peerInfo [dict get $messageDict socket remote] set peerAddress [lindex $peerInfo 0] set peerPort [lindex $peerInfo 1] set peer [dict create address $peerAddress port $peerPort] # XXX:TODO: Verify the nonce if {![info exists ::nano::node::_node_id_nonces($peer)]} { return "" } set sentNonce $::nano::node::_node_id_nonces($peer) unset ::nano::node::_node_id_nonces($peer) # Add the peer to our list of peers #puts "Got node_id_handshake response from $peer" set ::nano::node::peers($peer) [dict create lastSeen [clock seconds]] # Stats ::nano::node::stats::incr [list node_id_handshake response count] ::nano::node::stats::lappend [list node_id_handshake response uniqueKeys] [dict get $messageDict key] } return $retval } proc ::nano::network::server::confirm_req {messageDict} { ::nano::node::stats::incr [list confirm_req] return "" } proc ::nano::network::server::confirm_ack {messageDict} { # keep statistics dict with messageDict {} ::nano::node::stats::incr [list confirm_ack valid $valid] if {!$valid} { return "" } ::nano::node::stats::incr [list confirm_ack rep $voteAccount valid $valid] ::nano::node::stats::incr [list confirm_ack voteType $voteType] ::nano::node::stats::incr [list confirm_ack rep $voteAccount voteType $voteType] ::nano::node::stats::newMinMax [list confirm_ack rep $voteAccount voteSequence] $voteSequence set votedOn [llength $hashes] ::nano::node::stats::incr [list confirm_ack votedOnCount] $votedOn ::nano::node::stats::incr [list confirm_ack rep $voteAccount votedOnCount] $votedOn foreach hash $hashes { ::nano::node::stats::lappend [list confirm_ack votedOnUniqueCount] $hash ::nano::node::stats::lappend [list confirm_ack rep $voteAccount votedOnUniqueCount] $hash } return "" } proc ::nano::protocol::parse::publish {extensions messageData} { set blockTypeID [expr {($extensions >> 8) & 0x0f}] set blockType [::nano::block::typeFromTypeID $blockTypeID] set blockDict [::nano::block::dict::fromBlock $messageData -type=$blockType] |
︙ | ︙ | |||
3494 3495 3496 3497 3498 3499 3500 | } catch { set valid [::nano::internal::boolean [::nano::block::dict::verifySignature $block]] } set validWork [::nano::internal::boolean [::nano::block::dict::validateWork $block]] | | | | | < < | 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 | } catch { set valid [::nano::internal::boolean [::nano::block::dict::verifySignature $block]] } set validWork [::nano::internal::boolean [::nano::block::dict::validateWork $block]] ::nano::node::stats::incr [list publish valid $valid] ::nano::node::stats::incr [list publish validWork $validWork] ::nano::node::stats::incr [list publish type [dict get $block type]] ::nano::node::stats::lappend [list publish unique] $hash }]} { puts $::errorInfo } return "" } # Namespace ::nano::protocol::parse deals with the network level protocol (outside the node) # Namespace ::nano::network::server deals with the node's actual interaction with the network |
︙ | ︙ | |||
3714 3715 3716 3717 3718 3719 3720 | } proc ::nano::node::start args { package require defer package require udp set ::nano::node::startTime [clock seconds] | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 | } proc ::nano::node::start args { package require defer package require udp set ::nano::node::startTime [clock seconds] ::nano::node::stats::open array set config { -bootstrap true -realtime true -wait true } array set config $args if {$config(-bootstrap)} { coroutine ::nano::node::bootstrap::run ::nano::node::bootstrap } if {$config(-realtime)} { coroutine ::nano::node::realtime::run ::nano::node::realtime } if {$config(-wait)} { vwait ::nano::node::_FOREVER_ } } # Node stats proc ::nano::node::stats::open {} { set db ::nano::node::stats::_db if {[llength [info command $db]] != 0} { return } package require sqlite3 sqlite3 $db "" -create true tailcall clear } proc ::nano::node::stats::clear {} { set db ::nano::node::stats::_db $db eval {DROP TABLE IF EXISTS counters} $db eval {DROP TABLE IF EXISTS minmax} $db eval {DROP TABLE IF EXISTS lists} $db eval {DROP TABLE IF EXISTS info} $db eval {CREATE TABLE counters (key PRIMARY KEY, current INTEGER DEFAULT 0)} $db eval {CREATE TABLE minmax (key PRIMARY KEY, high INTEGER NOT NULL, low INTEGER NOT NULL)} $db eval {CREATE TABLE lists (key NOT NULL, value, UNIQUE (key, value))} $db eval {CREATE TABLE info (key PRIMARY KEY, value NOT NULL)} set now [clock seconds] $db eval {INSERT OR REPLACE INTO info (key, value) VALUES ('startTime', $now)} return } proc ::nano::node::stats::incr {key {amount 1}} { set db ::nano::node::stats::_db # Requires a new version of SQLite3 set haveUpsert false if {$haveUpsert} { # Untested set query { INSERT INTO counters (key, current) VALUES ($key, '@@AMOUNT@@') ON CONFLICT (key) DO UPDATE counters SET current = current + @@AMOUNT@@; } } else { set query { INSERT OR IGNORE INTO counters (key, current) VALUES ($key, '0'); UPDATE counters SET current = current + @@AMOUNT@@ WHERE key = $key; } } $db eval [string map [list @@AMOUNT@@ $amount] $query] } proc ::nano::node::stats::newMinMax {key value} { set db ::nano::node::stats::_db $db eval { INSERT OR IGNORE INTO minmax (key, high, low) VALUES ($key, $value, $value); UPDATE minmax SET high = $value WHERE key = $key AND $value > high; UPDATE minmax SET low = $value WHERE key = $key AND $value < low; } } proc ::nano::node::stats::lappend {key value} { set db ::nano::node::stats::_db $db eval {INSERT OR IGNORE INTO lists (key, value) VALUES ($key, $value)} } proc ::nano::node::stats::startTime {} { set db ::nano::node::stats::_db set startTime [$db onecolumn {SELECT value FROM info WHERE key = 'startTime'}] return $startTime } proc ::nano::node::stats::get {} { set db ::nano::node::stats::_db set results(startTime) [startTime] unset -nocomplain row $db eval {SELECT key, current FROM counters} row { set results($row(key)) $row(current) } unset -nocomplain row $db eval {SELECT key, COUNT(key) count FROM lists GROUP BY key} row { set results($row(key)) $row(count) } unset -nocomplain row $db eval {SELECT key, high, low FROM minmax} row { set results([concat $row(key) min]) $row(low) set results([concat $row(key) max]) $row(high) } return [array get results] } # RPC Client ## Side-effect: Sets ::nano::rpc::client::config proc ::nano::rpc::client::init args { package require http 2 if {![info exists ::nano::rpc::client::config]} { |
︙ | ︙ | |||
3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 | set balance [join [list $leading $trailing] "."] set balance [string trimright $balance "."] set result [list $balance $baseUnit] return $result } # Generic CLI helpers proc ::nano::_cli {namespace args} { for {set argIndex 0} {$argIndex < [llength $args]} {incr argIndex} { set arg [lindex $args $argIndex] switch -exact -- $arg { "-prompt" { | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 | set balance [join [list $leading $trailing] "."] set balance [string trimright $balance "."] set result [list $balance $baseUnit] return $result } # Node Wallet Functions proc ::nano::wallet::decode_backup {password walletJSON} { array set walletArray [::json::json2dict $walletJSON] set wallet(version) [expr 0x$walletArray(0000000000000000000000000000000000000000000000000000000000000000)] set wallet(salt) [binary decode hex $walletArray(0000000000000000000000000000000000000000000000000000000000000001)] set wallet(key) [binary decode hex $walletArray(0000000000000000000000000000000000000000000000000000000000000002)] set wallet(check) [binary decode hex $walletArray(0000000000000000000000000000000000000000000000000000000000000003)] set wallet(rep) [::nano::address::fromPublicKey $walletArray(0000000000000000000000000000000000000000000000000000000000000004)] set wallet(seed) [binary decode hex $walletArray(0000000000000000000000000000000000000000000000000000000000000005)] set wallet(index) [expr 0x$walletArray(0000000000000000000000000000000000000000000000000000000000000006)] if {$wallet(version) != 4} { return -code error "Unsupported wallet backup version ($version)" } set walletKeyIV [string range $wallet(salt) 0 15] set seedIV [string range $wallet(salt) 16 end] # XXX:TODO: Check the password against "check" # Decrypt seed set aesKey [::nano::internal::deriveKeyFromPassword $password $wallet(salt)] set walletKey [::nano::internal::AES256-CTR $aesKey $walletKeyIV $wallet(key)] set seed [::nano::internal::AES256-CTR $walletKey $seedIV $wallet(seed)] # Format results set wallet(seed) [string toupper [binary encode hex $seed]] # XXX:TODO: Include ad-hoc keys return [array get wallet] } # Generic CLI helpers proc ::nano::_cli {namespace args} { for {set argIndex 0} {$argIndex < [llength $args]} {incr argIndex} { set arg [lindex $args $argIndex] switch -exact -- $arg { "-prompt" { |
︙ | ︙ | |||
4078 4079 4080 4081 4082 4083 4084 | tailcall $proc {*}$args } proc {::nano::_cli::multiword help} {namespace base args} { tailcall help $namespace $base {*}$args } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 | tailcall $proc {*}$args } proc {::nano::_cli::multiword help} {namespace base args} { tailcall help $namespace $base {*}$args } # Node CLI proc ::nano::node::cli {args} { tailcall ::nano::_cli node -prompt { return "\[[dict get $::nano::node::configuration network]\] nano-node [package present nano]> " } {*}$args } proc {::nano::node::cli::show uptime} {} { set now [clock seconds] set start $::nano::node::startTime set statsStart [::nano::node::stats::startTime] set uptime [expr {$now - $start}] set uptimeStats [expr {$now - $statsStart}] set format {%-19s: %s} lappend response [format $format Uptime [::nano::_cli::interval $uptime]] lappend response [format $format "Stats last cleared" "[::nano::_cli::interval $uptimeStats] ago"] |
︙ | ︙ | |||
4145 4146 4147 4148 4149 4150 4151 | } proc {::nano::node::cli::show logs} args { return [join [lrange $::nano::node::log end-19 end] "\n"] } proc {::nano::node::cli::clear stats} args { | < < | < < > > > > > > < | | 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 | } proc {::nano::node::cli::show logs} args { return [join [lrange $::nano::node::log end-19 end] "\n"] } proc {::nano::node::cli::clear stats} args { ::nano::node::stats::clear return } proc {::nano::node::cli::clear peers} args { return -code error "Unimplemented" set ::nano::node::statsStartTime [clock seconds] unset -nocomplain ::nano::node::peers array set ::nano::node::peers [list] return } proc {::nano::node::cli::show stats} args { set stats [::nano::node::stats::get] set statsStart [dict get $stats startTime] dict unset stats startTime set now [clock seconds] set uptimeStats [expr {$now - $statsStart}] set quiet false if {!$quiet} { set format {%-19s: %s} puts [format $format "Stats last cleared" "[::nano::_cli::interval $uptimeStats] ago"] puts "" } set globalOnly false if {[lindex $args 0] eq "-global"} { set globalOnly true } if {[lindex $args 0] in {"-rep" "-representative"}} { set repOnly [::nano::address::toPublicKey [lindex $args 1] -binary] } set maxKeyLen 0 foreach {key val} $stats { if {[lindex $key 1] eq "rep"} { if {$globalOnly} { continue } elseif {[info exists repOnly]} { if {$repOnly ne [::nano::address::toPublicKey [lindex $key 2] -binary]} { continue } |
︙ | ︙ | |||
4215 4216 4217 4218 4219 4220 4221 | foreach {key val} [lsort -stride 2 -dictionary [array get localStats]] { set extra "" if {![regexp { (min|max)[A-Z]} $key]} { if {[string is entier -strict $val]} { set valAvg [expr {($val * 1.0) / $uptimeStats}] set valAvg [format %.4f $valAvg] | > > | > | 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 | foreach {key val} [lsort -stride 2 -dictionary [array get localStats]] { set extra "" if {![regexp { (min|max)[A-Z]} $key]} { if {[string is entier -strict $val]} { set valAvg [expr {($val * 1.0) / $uptimeStats}] set valAvg [format %.4f $valAvg] if {[lindex $key end] ni {max min}} { set extra " (avg: $valAvg per second)" } } } puts [format "%-${maxKeyLen}s = %s%s" $key $val $extra] } return |
︙ | ︙ |