Check-in [18d1a69d74]
Overview
Comment:More work on lazy bootstrapping testbed
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 18d1a69d74e63ac7aedada85340f140878bc107dbac23f82241d4320d8b8bc71
User & Date: rkeene on 2018-12-08 02:31:38
Other Links: manifest | tags
Context
2018-12-08
02:33
Added node script check-in: 9d3190fba9 user: rkeene tags: trunk
02:31
More work on lazy bootstrapping testbed check-in: 18d1a69d74 user: rkeene tags: trunk
2018-12-05
19:25
Added CLI stuff and a bit of cleanup check-in: 16f85b0e66 user: rkeene tags: trunk
Changes

Modified nano.tcl from [23ff6024a7] to [984f7c64c2].

27
28
29
30
31
32
33
















34
35
36
37
38
39
40
namespace eval ::nano::network::server {}
namespace eval ::nano::protocol::create {}
namespace eval ::nano::protocol::parse {}
namespace eval ::nano::protocol::extensions {}
namespace eval ::nano::network::_dns {}

# Constants
















set ::nano::block::stateBlockPreamble [binary decode hex "0000000000000000000000000000000000000000000000000000000000000006"]
set ::nano::block::zero "0000000000000000000000000000000000000000000000000000000000000000"
set ::nano::balance::zero "00000000000000000000000000000000"
set ::nano::address::zero $::nano::block::zero
set ::nano::address::base32alphabet {13456789abcdefghijkmnopqrstuwxyz}
set ::nano::network::messageTypes {
	"invalid"







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
namespace eval ::nano::network::server {}
namespace eval ::nano::protocol::create {}
namespace eval ::nano::protocol::parse {}
namespace eval ::nano::protocol::extensions {}
namespace eval ::nano::network::_dns {}

# Constants
set ::nano::block::genesis(main) {{
	"type": "open",
	"source": "E89208DD038FBB269987689621D52292AE9C35941A7484756ECCED92A65093BA",
	"representative": "xrb_3t6k35gi95xu6tergt6p69ck76ogmitsa8mnijtpxm9fkcm736xtoncuohr3",
	"account": "xrb_3t6k35gi95xu6tergt6p69ck76ogmitsa8mnijtpxm9fkcm736xtoncuohr3",
	"work": "62f05417dd3fb691",
	"signature": "9F0C933C8ADE004D808EA1985FA746A7E95BA2A38F867640F53EC8F180BDFE9E2C1268DEAD7C2664F356E37ABA362BC58E46DBA03E523A7B5A19E4B6EB12BB02"
}}
set ::nano::block::genesis(beta) {{
	"type": "open",
	"source": "A59A47CC4F593E75AE9AD653FDA9358E2F7898D9ACC8C60E80D0495CE20FBA9F",
	"representative": "xrb_3betaz86ypbygpqbookmzpnmd5jhh4efmd8arr9a3n4bdmj1zgnzad7xpmfp",
	"account": "xrb_3betaz86ypbygpqbookmzpnmd5jhh4efmd8arr9a3n4bdmj1zgnzad7xpmfp",
	"work": "000000000f0aaeeb",
	"signature": "A726490E3325E4FA59C1C900D5B6EEBB15FE13D99F49D475B93F0AACC5635929A0614CF3892764A04D1C6732A0D716FFEB254D4154C6F544D11E6630F201450B"
}}
set ::nano::block::stateBlockPreamble [binary decode hex "0000000000000000000000000000000000000000000000000000000000000006"]
set ::nano::block::zero "0000000000000000000000000000000000000000000000000000000000000000"
set ::nano::balance::zero "00000000000000000000000000000000"
set ::nano::address::zero $::nano::block::zero
set ::nano::address::base32alphabet {13456789abcdefghijkmnopqrstuwxyz}
set ::nano::network::messageTypes {
	"invalid"
79
80
81
82
83
84
85



86
87
88
89
90
91
92
	raw   1
}
set ::nano::protocol::extensions {
	node_id_handshake {
		query 1
		response 2
	}



}

# Address management functions
proc ::nano::address::toPublicKey {address args} {
	set performChecksumCheck false
	set outputFormat "bytes"
	foreach arg $args {







>
>
>







95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
	raw   1
}
set ::nano::protocol::extensions {
	node_id_handshake {
		query 1
		response 2
	}
	bulk_pull {
		count_present 1
	}
}

# Address management functions
proc ::nano::address::toPublicKey {address args} {
	set performChecksumCheck false
	set outputFormat "bytes"
	foreach arg $args {
1073
1074
1075
1076
1077
1078
1079












1080
1081
1082
1083
1084
1085
1086
		}

		return -level 0 true
	}]
	set blockJSON [::nano::block::json::fromDict $blockDict]
	return $blockJSON
}













#   send from <account> to <account> previousBalance <balance>
#        amount <amount> sourceBlock <sourceBlockHash>
#        previous <previousBlockHash> ?representative <representative>?
proc ::nano::block::create::send {args} {
	array set block $args
	if {![info exists block(representative)]} {







>
>
>
>
>
>
>
>
>
>
>
>







1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
		}

		return -level 0 true
	}]
	set blockJSON [::nano::block::json::fromDict $blockDict]
	return $blockJSON
}

proc ::nano::block::json::genesis {network} {
	return $::nano::block::genesis($network)
}

proc ::nano::block::dict::genesis {network} {
	if {![info exists ::nano::block::dict::genesis($network)]} {
		set ::nano::block::dict::genesis($network) [::nano::block::dict::fromJSON [::nano::block::json::genesis $network]]
	}

	return $::nano::block::dict::genesis($network)
}

#   send from <account> to <account> previousBalance <balance>
#        amount <amount> sourceBlock <sourceBlockHash>
#        previous <previousBlockHash> ?representative <representative>?
proc ::nano::block::create::send {args} {
	array set block $args
	if {![info exists block(representative)]} {
1688
1689
1690
1691
1692
1693
1694
1695



1696
1697
1698
1699
1700
1701
1702
1703
1704
		}
	}

	return
}

# Node Configuration
proc ::nano::node::_configDictToJSON {configDict {prefix ""}} {



	set values [list]
	foreach key [dict keys $configDict] {
		set value [dict get $configDict $key]
		switch -- ${prefix}${key} {
			"rpc" - "node" - "opencl" - "node/logging" - "node/database" {
				set value [_configDictToJSON $value "$key/"]
			}
			"node/preconfigured_peers" - "node/preconfigured_representatives" - "node/work_peers" {
				set value [json::write array {*}[lmap item $value {







|
>
>
>

|







1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
		}
	}

	return
}

# Node Configuration
proc ::nano::node::_configDictToJSON {configDict {prefix ""} {keyPattern ""}} {
	if {$keyPattern eq ""} {
		set keyPattern "*"
	}
	set values [list]
	foreach key [dict keys $configDict $keyPattern] {
		set value [dict get $configDict $key]
		switch -- ${prefix}${key} {
			"rpc" - "node" - "opencl" - "node/logging" - "node/database" {
				set value [_configDictToJSON $value "$key/"]
			}
			"node/preconfigured_peers" - "node/preconfigured_representatives" - "node/work_peers" {
				set value [json::write array {*}[lmap item $value {
1902
1903
1904
1905
1906
1907
1908
1909




















1910




1911
1912

1913
1914
1915
1916
1917
1918
1919
1920
1921
1922

1923














































































































1924
1925
1926
1927
1928
1929
1930
	upvar 1 $extensionsVar extensions

	::set value [dict get $::nano::protocol::extensions $messageType $flag]
	::set extensions [expr {$extensions & ((~$value) & 0xffff)}]
}


proc ::nano::protocol::create::bulk_pull {account {end ""}} {




















	set accountPubKey [::nano::address::toPublicKey $account -binary]





	if {$end ne ""} {

		if {[string length $end] != $::nano::block::hashLength} {
			set end [binary decode hex $end]
		}
	} else {
		set end [binary decode hex $::nano::block::zero]
	}

	return [dict create data [binary format a32a32 \
		$accountPubKey \
		$end \

	]]














































































































}

proc ::nano::protocol::create::_bulk_pull_account_flagsParse {direction flag} {
	switch -- $direction {
		"toInt" {
			switch -- $flag {
				"" - "default" - "pendingHashAndAmount" {







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
|
|
>







|


>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
	upvar 1 $extensionsVar extensions

	::set value [dict get $::nano::protocol::extensions $messageType $flag]
	::set extensions [expr {$extensions & ((~$value) & 0xffff)}]
}


proc ::nano::protocol::create::bulk_pull {account args} {
	set count 0
	set extensions 0
	foreach {arg argVal} $args {
		switch -exact -- $arg {
			"-end" {
				set end $argVal
			}
			"-count" {
				set count $argVal
			}
			"-partialOkay" - "-foreach" {
				# Ignored, used by response parser
			}
			default {
				error "Invalid option: $arg"
			}
		}
	}

	if {[string length $account] in {64 65} && [lindex [split $account _] 0] in {xrb nano}} {
		set accountPubKey [::nano::address::toPublicKey $account -binary]
	} elseif {[string length $account] != $::nano::key::publicKeyLength} {
		set accountPubKey [binary decode hex $account]
	} else {
		set accountPubKey $account
	}

	if {[info exists end]} {
		if {[string length $end] != $::nano::block::hashLength} {
			set end [binary decode hex $end]
		}
	} else {
		set end [binary decode hex $::nano::block::zero]
	}

	set request [binary format a32a32 \
		$accountPubKey \
		$end \
	]

	if {$count != 0} {
		::nano::protocol::extensions::set "bulk_pull" extensions "count_present"
		append request [binary format ciccc 0 $count 0 0 0]
	} else {
		::nano::protocol::extensions::unset "bulk_pull" extensions "count_present"
	}

	return [dict create extensions $extensions data $request]
}

proc ::nano::protocol::parse::bulk_pull_response {sock account args} {
	set partialOkay false
	foreach {arg argVal} $args {
		switch -exact -- $arg {
			"-end" {
				# Ignored, used by request creator
			}
			"-partialOkay" {
				set partialOkay $argVal
			}
			"-foreach" {
				set blockVar [lindex $argVal 0]
				set blockHandler [lindex $argVal 1]
			}
		}
	}

	set blocks [list]

	set success false

	while true {
		if {[catch {
			set type [::nano::network::_recv $sock 1]
		} err]} {
			::nano::node::log "Error from $sock: $err"
			break
		}

		set type [::nano::block::typeFromTypeID $type -binary]

		if {$type eq "not_a_block"} {
			if {[info exists blockHandler]} {
				uplevel 2 [list set $blockVar ""]
				catch [list uplevel 2 $blockHandler]
			}

			set success true

			break
		}

		set length [::nano::block::lengthFromType $type -work -signature]

		if {[catch {
			set block [::nano::network::_recv $sock $length]
		} err]} {
			::nano::node::log "Error from $sock: $err"
			break
		}

		set block [::nano::block::dict::fromBlock $block -type=$type]

		if {[info exists blockHandler]} {
			uplevel 2 [list set $blockVar $block]
			set blockHandlerCode [catch [list uplevel 2 $blockHandler] blockHandlerResult blockHandlerOptions]
			switch -exact -- $blockHandlerCode {
				0 {}
				1 { return {*}$blockHandlerOptions $blockHandlerResult }
				2 { error "Not implemented" }
				3 {
					set success true

					break
				}
				4 { continue }
			}
		} else {
			lappend blocks $block
		}
	}

	if {!$success} {
		if {!$partialOkay} {
			error "Failed to pull from $sock"
		} elseif {[info exists blockHandler]} {
			uplevel 2 [list set $blockVar "error"]
			catch [list uplevel 2 $blockHandler]
		}
	}

	return $blocks
}

proc ::nano::protocol::parse::bulk_pull {extensions messageData} {
	set flags [::nano::protocol::extensions::get "bulk_pull" $extensions]

	binary scan $messageData H64H64a* start end extra
	set retval [dict create start $start]

	if {$end ni $::nano::block::zero} {
		dict set retval end $end
	}

	if {"count_present" in $flags} {
		binary scan $extra cuic* _ count _
		dict set retval count $count
	}

	return $retval
}

proc ::nano::protocol::create::_bulk_pull_account_flagsParse {direction flag} {
	switch -- $direction {
		"toInt" {
			switch -- $flag {
				"" - "default" - "pendingHashAndAmount" {
2412
2413
2414
2415
2416
2417
2418

























































































































































































































































2419
2420
2421
2422
2423
2424
2425
	set response ""
	if {[info command $responseCommand] ne ""} {
		set response [$responseCommand $sock {*}$args]
	}

	return $response
}


























































































































































































































































proc ::nano::node::bootstrap::peer {peer peerPort} {
	::nano::node::log "Connecting to ${peer}:${peerPort}"

	catch {
		set sock [::nano::network::_connect $peer $peerPort]
	} err







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
	set response ""
	if {[info command $responseCommand] ne ""} {
		set response [$responseCommand $sock {*}$args]
	}

	return $response
}

proc ::nano::node::bootstrap::TMP_LEDGER_ADDBLOCK {block} {
	set blockHash [::nano::block::dict::toHash $block -binary]
	set ::nano::node::bootstrap::TMP_LEDGER($blockHash) 1
}

proc ::nano::node::bootstrap::TMP_LEDGER_ADDBLOCKHASH {blockHash} {
	if {[string length $blockHash] != $::nano::block::hashLength} {
		set blockHash [binary decode hex $blockHash]
	}

	set ::nano::node::bootstrap::TMP_LEDGER($blockHash) 1
}

proc ::nano::node::bootstrap::TMP_LEDGER_CHECKBLOCKHASH {blockHash} {
	if {[string length $blockHash] != $::nano::block::hashLength} {
		set blockHash [binary decode hex $blockHash]
	}

	if {[info exists ::nano::node::bootstrap::TMP_LEDGER($blockHash)]} {
		return true
	}

	return false
}

proc ::nano::node::bootstrap::TMP_LEDGER_BLOCKHASHCOUNT {} {
	return [llength [array names ::nano::node::bootstrap::TMP_LEDGER]]
}

proc ::nano::node::bootstrap::lazy_peer {peerAddress peerPort} {
	incr ::nano::node::bootstrap::lazyPeerCount

	set peerString "$peerAddress $peerPort"

	set firstTime true
	while {[llength [array names ::nano::node::bootstrap::lazy_pullQueue]] != 0} {
		if {!$firstTime && ![info exists peerSock]} {
			incr ::nano::node::bootstrap::lazyPeerCount -1

			return
		}

		set blockHash [lindex [array names ::nano::node::bootstrap::lazy_pullQueue] 0]
		set blockHashString [string toupper [binary encode hex $blockHash]]

		if {[::nano::node::bootstrap::TMP_LEDGER_CHECKBLOCKHASH $blockHash]} {
			unset -nocomplain ::nano::node::bootstrap::lazy_pullQueue($blockHash)
			::nano::node::log "Skipping $blockHashString (already in local ledger -- this is okay, it just means we queued something but by the time it was acted on it was already fetched, so no work is needed)"

			# Avoid getting busy in this loop and monopolizing the thread
			# by yielding temporarily
			::nano::node::_sleep 0

			continue
		}

		if {$firstTime} {
			set firstTime false

			::nano::node::log "Connecting to $peerString"

			if {[catch {
				set peerSock [::nano::node::createSocket bootstrap $peerAddress $peerPort]
			}]} {
				::nano::node::log "Failed to connect to $peerString"

				incr ::nano::node::bootstrap::lazyPeerCount -1

				return
			}
		}

		::nano::node::log "Pulling $blockHashString from $peerString"

		unset -nocomplain chainAccount chainFrontier block
		set origBlockHash $blockHash
		set lastBlock ""
		set failedToDownload false

		if {[catch {
			set chainReachedEnd false
			::nano::network::client $peerSock bulk_pull $blockHash -count 128 -foreach {prevBlock {
				set isFinalBlock false
				if {$prevBlock eq ""} {
					set isFinalBlock true
				}

				# We don't start processing blocks until after we have pulled the second block
				# so that we have access to the next block in the download (which is the
				# previous block in the chain) to verify balance changes for state blocks
				set block $lastBlock
				set lastBlock $prevBlock
				if {$block eq ""} {
					continue
				}

				set blockType [dict get $block type]
				set blockHash [::nano::block::dict::toHash $block -binary]
				set blockHashString [string toupper [binary encode hex $blockHash]]

				# Remove this block from the pull queue
				unset -nocomplain ::nano::node::bootstrap::lazy_pullQueue($blockHash)

				# Get the frontier and account for this chain
				if {![info exists chainFrontier]} {
					set chainFrontier $blockHash
				}

				if {![info exists chainAccount]} {
					if {[dict exists $block "account"]} {
						set chainAccount [dict get $block "account"]
						set seenFrontiers($chainAccount) $chainFrontier
					}
				}

				# Determine if this block is a receiving block
				unset -nocomplain isReceive link
				if {$blockType in {open receive}} {
					set isReceive true
				}

				if {$blockType in {send change}} {
					set isReceive false
				}

				if {![info exists isReceive]} {
					if {$prevBlock eq ""} {
						if {$blockType eq "state"} {
							if {[dict get $block previous] eq $::nano::block::zero} {
								set isReceive true
							} else {
								set ::nano::node::bootstrap::lazy_pullQueue([binary decode hex [dict get $block previous]]) true
							}
						} else {
							error "Something is wrong.  There is no following block to this $blockType block"
						}
					}
				}

				if {![info exists isReceive]} {
					catch {
						if {[dict get $block balance] > [dict get $prevBlock balance]} {
							set isReceive true
						}
					}
				}

				if {![info exists isReceive]} {
					set isReceive false
				}

				# If so, find the source block hash
				if {$isReceive} {
					switch -- $blockType {
						"state" {
							set link [dict get $block link]
						}
						"open" - "receive" {
							set link [dict get $block source]
						}
						default {
							error "Unable to handle receiving from block type $blockType"
						}
					}

					set link [string toupper $link]
				}

				# Determine if this block is an open block
				set isOpen false
				if {$blockType eq "open"} {
					set isOpen true
				} elseif {$blockType eq "state" && [dict get $block previous] eq $::nano::block::zero} {
					set isOpen true
				}

				# If we reached the open block, declare victory
				if {$isOpen} {
					set chainReachedEnd true
				}

				# Determine if we already have this block
				set alreadyPulledLink false

				if {[::nano::node::bootstrap::TMP_LEDGER_CHECKBLOCKHASH $blockHash]} {
					set alreadyPulledThisBlock true
				} else {
					set alreadyPulledThisBlock false

					if {$isReceive && [::nano::node::bootstrap::TMP_LEDGER_CHECKBLOCKHASH $link]} {
						set alreadyPulledLink true
					}
				}

				::nano::node::bootstrap::TMP_LEDGER_ADDBLOCK $block

				set skippingText ""
				if {$alreadyPulledLink} {
					set skippingText " (skipped adding link)"
				}

				if {$alreadyPulledThisBlock} {
					set skippingText " (already have this block, thus the rest of this chain)"
				}

				if {$isReceive} {
					::nano::node::log "Pull got $blockHashString ($blockType) <- $link${skippingText}"

					if {!$alreadyPulledThisBlock && !$alreadyPulledLink} {
						set ::nano::node::bootstrap::lazy_pullQueue([binary decode hex $link]) true
					}
				} else {
					::nano::node::log "Pull got $blockHashString ($blockType)${skippingText}"
				}
			}}

			if {!$chainReachedEnd} {
				set previous [dict get $block previous]
				set ::nano::node::bootstrap::lazy_pullQueue([binary decode hex $previous]) true
			}
		} bulkPullError]} {
			set failedToDownload true
		}

		if {$failedToDownload} {
			::nano::node::log "Error: $bulkPullError"

			if {[info exists peerSock]} {
				::nano::node::closeSocket $peerSock
				unset peerSock
			}

			if {$lastBlock ne ""} {
				set blockHash [::nano::block::dict::toHash $block -binary]
			} else {
				set blockHash $origBlockHash
			}

			set ::nano::node::bootstrap::lazy_pullQueue($origBlockHash) true
		}
	}

	if {[info exists peerSock]} {
		::nano::node::closeSocket $peerSock
	}

	incr ::nano::node::bootstrap::lazyPeerCount -1
}

proc ::nano::node::bootstrap::peer {peer peerPort} {
	::nano::node::log "Connecting to ${peer}:${peerPort}"

	catch {
		set sock [::nano::network::_connect $peer $peerPort]
	} err
2519
2520
2521
2522
2523
2524
2525









































2526
2527
2528
2529
2530
2531
2532
		if {$a < $b} {
			return -1
		} else {
			return 1
		}
	}} $salt] $list
}










































proc ::nano::node::bootstrap {} {
	set ::nano::node::bootstrap::frontiers_to_pull [list]
	set ::nano::node::bootstrap::frontier_req_running false

	while true {
		set peerInfoList [::nano::node::getPeers]







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
		if {$a < $b} {
			return -1
		} else {
			return 1
		}
	}} $salt] $list
}

proc ::nano::node::bootstrap_lazy {startBlockHash} {
	if {[string length $startBlockHash] != $::nano::block::hashLength} {
		set startBlockHash [binary decode hex $startBlockHash]
	}

	set ::nano::node::bootstrap::lazy_pullQueue($startBlockHash) true

	if {[info exists ::nano::node::bootstrap::lazyRunning]} {
		return
	}
	set ::nano::node::bootstrap::lazyRunning true

	set ::nano::node::bootstrap::lazyPeerCount 0

	while {[llength [array names ::nano::node::bootstrap::lazy_pullQueue]] != 0 || $::nano::node::bootstrap::lazyPeerCount != 0} {
		set peerInfoList [::nano::node::getPeers]
		::nano::node::log "Have [llength $peerInfoList] peers"

		foreach peerInfo $peerInfoList {
			set peer     [dict get $peerInfo "address"]
			set peerPort [dict get $peerInfo "port"]

			if {[llength [info command ::nano::node::bootstrap::peer_*]] >= [dict get $::nano::node::configuration node bootstrap_connections]} {
				continue
			}

			set peerId [binary encode hex [::nano::internal::hashData "lazy:$peer:$peerPort" 5]]

			if {[info command ::nano::node::bootstrap::peer_${peerId}] ne ""} {
				continue
			}

			coroutine ::nano::node::bootstrap::peer_${peerId} ::nano::node::bootstrap::lazy_peer $peer $peerPort
		}

		::nano::node::_sleep 30000
	}

	unset ::nano::node::bootstrap::lazyRunning
}

proc ::nano::node::bootstrap {} {
	set ::nano::node::bootstrap::frontiers_to_pull [list]
	set ::nano::node::bootstrap::frontier_req_running false

	while true {
		set peerInfoList [::nano::node::getPeers]
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567

		::nano::node::_sleep 30000
	}
}

proc ::nano::network::_connect {host port} {
	if {[info coroutine] eq ""} {
		set sock [::socket $host $port]
	} else {
		set sock [::socket -async $host $port]
		chan event $sock writable [info coroutine]
		chan event $sock readable [info coroutine]

		if {![chan configure $sock -connecting]} {
			if {[chan configure $sock -error] ne ""} {
				close $sock








|

|







3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027

		::nano::node::_sleep 30000
	}
}

proc ::nano::network::_connect {host port} {
	if {[info coroutine] eq ""} {
		set sock [socket $host $port]
	} else {
		set sock [socket -async $host $port]
		chan event $sock writable [info coroutine]
		chan event $sock readable [info coroutine]

		if {![chan configure $sock -connecting]} {
			if {[chan configure $sock -error] ne ""} {
				close $sock

2631
2632
2633
2634
2635
2636
2637



2638
2639

2640
2641
2642
2643
2644
2645
2646
}

proc ::nano::node::_sleep {ms {verbose 1}} {
	if {$verbose} {
		::nano::node::log "Sleeping for $ms ms"
	}




	after $ms [info coroutine]
	yield

}

proc ::nano::node::getPeers {} {
	if {[info exists ::nano::node::configuration]} {
		set peers [dict get $::nano::node::configuration node preconfigured_peers]
		set defaultPeerPort [dict get $::nano::node::configuration node peering_port]
	} else {







>
>
>
|
|
>







3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
}

proc ::nano::node::_sleep {ms {verbose 1}} {
	if {$verbose} {
		::nano::node::log "Sleeping for $ms ms"
	}

	if {[info coroutine] eq ""} {
		after $ms
	} else {
		after $ms [info coroutine]
		yield
	}
}

proc ::nano::node::getPeers {} {
	if {[info exists ::nano::node::configuration]} {
		set peers [dict get $::nano::node::configuration node preconfigured_peers]
		set defaultPeerPort [dict get $::nano::node::configuration node peering_port]
	} else {
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749

		# If this peer is not already known, contact them requesting a handshake
		if {![info exists ::nano::node::peers($peer)]} {
			if {![info exists ::nano::node::_node_id_nonces($peer)]} {
				set node_id_nonce [::nano::internal::randomBytes 32]
				set ::nano::node::_node_id_nonces($peer) [dict create query $node_id_nonce lastSeen $now]

				set peerSock [::nano::node::socket realtime $address $port]
#puts "Querying $peerSock with node_id_handshake (1)"
				::nano::network::client $peerSock "node_id_handshake" query -query $node_id_nonce
			}
		}
	}

	return ""







|







3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213

		# If this peer is not already known, contact them requesting a handshake
		if {![info exists ::nano::node::peers($peer)]} {
			if {![info exists ::nano::node::_node_id_nonces($peer)]} {
				set node_id_nonce [::nano::internal::randomBytes 32]
				set ::nano::node::_node_id_nonces($peer) [dict create query $node_id_nonce lastSeen $now]

				set peerSock [::nano::node::createSocket realtime $address $port]
#puts "Querying $peerSock with node_id_handshake (1)"
				::nano::network::client $peerSock "node_id_handshake" query -query $node_id_nonce
			}
		}
	}

	return ""
2943
2944
2945
2946
2947
2948
2949













2950
2951
2952
2953
2954
2955
2956
2957
			::nano::node::log "Error handling ${messageType}: $err"
		}
	}

	return $retval
}














proc ::nano::node::socket {type address port} {
	if {$type eq "realtime"} {
		# Determine our listening port
		set peeringPort [dict get $::nano::node::configuration node peering_port]

		# Start a UDP listening socket
		foreach protocolVersion {v4 v6} {
			if {[info exists ::nano::node::realtime::socket($protocolVersion)]} {







>
>
>
>
>
>
>
>
>
>
>
>
>
|







3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
			::nano::node::log "Error handling ${messageType}: $err"
		}
	}

	return $retval
}

proc ::nano::node::closeSocket {peerSock} {
	if {[catch {
		if {[dict get $peerSock "type"] eq "realtime"} {
			# XXX:TODO: Probably don't close this socket
			set socket [dict get $peerSock "socket"]
		}
	}]} {
		set socket $peerSock
	}

	close $socket
}

proc ::nano::node::createSocket {type address port} {
	if {$type eq "realtime"} {
		# Determine our listening port
		set peeringPort [dict get $::nano::node::configuration node peering_port]

		# Start a UDP listening socket
		foreach protocolVersion {v4 v6} {
			if {[info exists ::nano::node::realtime::socket($protocolVersion)]} {
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
	if {$data eq ""} {
		return
	}

	set remote [chan configure $socket -peer]
	set address [lindex $remote 0]
	set port [lindex $remote 1]
	set peerSock [::nano::node::socket realtime $address $port]
	set response [::nano::network::server $data "realtime" $peerSock]
	if {$response eq ""} {
		return
	}

	set response [dict get $response "invoke_client"]

	::nano::network::client $peerSock {*}$response

	return
}

proc ::nano::node::realtime {} {
	package require udp

	set clientIDPrivateKey [dict get $::nano::node::configuration node client_id_private_key]

	# Start listening
	::nano::node::socket realtime "" ""

	# Periodically send keepalives to all known peers
	while true {
		set allPeers [::nano::node::getPeers]
		set peers [lrange $allPeers 0 15]

		# XXX:TODO: Make this a configuration option







|


















|







3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
	if {$data eq ""} {
		return
	}

	set remote [chan configure $socket -peer]
	set address [lindex $remote 0]
	set port [lindex $remote 1]
	set peerSock [::nano::node::createSocket realtime $address $port]
	set response [::nano::network::server $data "realtime" $peerSock]
	if {$response eq ""} {
		return
	}

	set response [dict get $response "invoke_client"]

	::nano::network::client $peerSock {*}$response

	return
}

proc ::nano::node::realtime {} {
	package require udp

	set clientIDPrivateKey [dict get $::nano::node::configuration node client_id_private_key]

	# Start listening
	::nano::node::createSocket realtime "" ""

	# Periodically send keepalives to all known peers
	while true {
		set allPeers [::nano::node::getPeers]
		set peers [lrange $allPeers 0 15]

		# XXX:TODO: Make this a configuration option
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
			set contactPeers [lrange $allPeers 0 $peerNotifyCount]
		}

		foreach peerInfo $contactPeers {
			set peerAddress [dict get $peerInfo "address"]
			set peerPort [dict get $peerInfo "port"]

			set peerSock [::nano::node::socket realtime $peerAddress $peerPort]

			set node_id_nonce [::nano::internal::randomBytes 32]

			::nano::network::client $peerSock keepalive $peers -local true
#puts "Querying $peerSock with node_id_handshake (2)"
			::nano::network::client $peerSock node_id_handshake query -query $node_id_nonce
		}







|







3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
			set contactPeers [lrange $allPeers 0 $peerNotifyCount]
		}

		foreach peerInfo $contactPeers {
			set peerAddress [dict get $peerInfo "address"]
			set peerPort [dict get $peerInfo "port"]

			set peerSock [::nano::node::createSocket realtime $peerAddress $peerPort]

			set node_id_nonce [::nano::internal::randomBytes 32]

			::nano::network::client $peerSock keepalive $peers -local true
#puts "Querying $peerSock with node_id_handshake (2)"
			::nano::network::client $peerSock node_id_handshake query -query $node_id_nonce
		}
3319
3320
3321
3322
3323
3324
3325




3326
3327
3328

3329








3330
3331
3332
3333
3334
3335
3336
3337
		return
	}

	set globalOnly false
	if {[lindex $args 0] eq "-global"} {
		set globalOnly true
	}





	set maxKeyLen 0
	foreach {key val} [array get ::nano::node::stats] {

		if {$globalOnly} {








			if {[lindex $key 1] eq "rep"} {
				continue
			}
		}

		set keyLen [string length $key]
		if {$keyLen > $maxKeyLen} {
			set maxKeyLen $keyLen







>
>
>
>



>
|
>
>
>
>
>
>
>
>
|







3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
		return
	}

	set globalOnly false
	if {[lindex $args 0] eq "-global"} {
		set globalOnly true
	}

	if {[lindex $args 0] in {"-rep" "-representative"}} {
		set repOnly [::nano::address::toPublicKey [lindex $args 1] -binary]
	}

	set maxKeyLen 0
	foreach {key val} [array get ::nano::node::stats] {
		if {[lindex $key 1] eq "rep"} {
			if {$globalOnly} {
				continue
			} elseif {[info exists repOnly]} {
				if {$repOnly ne [::nano::address::toPublicKey [lindex $key 2] -binary]} {
					continue
				}
				set key [concat [lrange $key 0 0] [lrange $key 3 end]]
			}
		} else {
			if {[info exists repOnly]} {
				continue
			}
		}

		set keyLen [string length $key]
		if {$keyLen > $maxKeyLen} {
			set maxKeyLen $keyLen
3354
3355
3356
3357
3358
3359
3360




3361
3362
3363
3364
3365
3366
3367





















3368
3369
3370
3371
3372
3373
3374
proc ::nano::node::cli::help args {
	set response [list]
	if {[llength $args] == 0} {
		lappend response "Commands:"

		foreach command [lsort -dictionary [info command ::nano::node::cli::*]] {
			set command [namespace tail $command]




			set description ""
			lappend response [format "   %-12s - %s" $command $description]
		}
	}

	return [join $response "\n"]
}






















proc ::nano::node::cli::peers args {
	if {[lindex $args 0] eq "-count"} {
		return [llength [::nano::node::getPeers]]
	}

	if {[llength $args] == 1} {







>
>
>
>







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
proc ::nano::node::cli::help args {
	set response [list]
	if {[llength $args] == 0} {
		lappend response "Commands:"

		foreach command [lsort -dictionary [info command ::nano::node::cli::*]] {
			set command [namespace tail $command]
			if {[string match "_*" $command]} {
				continue
			}

			set description ""
			lappend response [format "   %-12s - %s" $command $description]
		}
	}

	return [join $response "\n"]
}

proc ::nano::node::cli::network {} {
	return [dict get $::nano::node::configuration network]
}

proc ::nano::node::cli::config args {
	set config $::nano::node::configuration
	dict set config node client_id_private_key [binary encode hex [dict get $config node client_id_private_key]]
	set subtreePrefix [lrange $args 0 end-1]
	set subtreeKey [lindex $args end]
	set subtreePrefixConfig [join $subtreePrefix /]
	if {$subtreePrefixConfig ne ""} {
		append subtreePrefixConfig "/"
	}

	set subtree [dict get $config {*}$subtreePrefix]

	set json [::nano::node::_configDictToJSON $subtree $subtreePrefixConfig $subtreeKey]

	return $json
}

proc ::nano::node::cli::peers args {
	if {[lindex $args 0] eq "-count"} {
		return [llength [::nano::node::getPeers]]
	}

	if {[llength $args] == 1} {
3402
3403
3404
3405
3406
3407
3408






































3409
3410
3411
3412
3413
3414
3415
3416
			set age "statically configured peer"
		}

		lappend response "  $peer: $age"
	}
	return [join $response "\n"]
}







































namespace eval ::nano::node::cli {
	namespace export -clear *
}

if {[info exists ::nano::node::cli::_using_repl]} {
	::nano::node::cli -import
}







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>








3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
			set age "statically configured peer"
		}

		lappend response "  $peer: $age"
	}
	return [join $response "\n"]
}

proc ::nano::node::cli::_pull_chain {startBlockHash {endBlockHash "genesis"}} {
	if {$endBlockHash eq "genesis"} {
		set network [dict get $::nano::node::configuration network]
		set endBlockHash [::nano::block::dict::toHash [::nano::block::dict::genesis $network] -hex]
	}

	set startBlockHash [binary decode hex $startBlockHash]
	set endBlockHash [binary decode hex $endBlockHash]

	if {[string length $startBlockHash] != $::nano::block::hashLength} {
		error "Invalid start block hash specified"
	}

	if {[string length $endBlockHash] != $::nano::block::hashLength} {
		error "Invalid end block hash specified"
	}

	set startBlockHash [string toupper [binary encode hex $startBlockHash]]
	set endBlockHash [string toupper [binary encode hex $endBlockHash]]

	::nano::node::bootstrap::TMP_LEDGER_ADDBLOCKHASH $endBlockHash

	puts "Pulling $startBlockHash to $endBlockHash"

	set startTime [clock seconds]
	coroutine ::nano::node::bootstrap::runLazy ::nano::node::bootstrap_lazy $startBlockHash
	if {![info exists ::nano::node::bootstrap::lazyRunning]} {
		vwait ::nano::node::bootstrap::lazyRunning
	}
	vwait ::nano::node::bootstrap::lazyRunning
	set endTime [clock seconds]

	set blocksPulled [::nano::node::bootstrap::TMP_LEDGER_BLOCKHASHCOUNT]
	set delta [expr {$endTime - $startTime}]

	puts "Pulled $blocksPulled blocks in [::nano::node::cli::_interval $delta]"
}

namespace eval ::nano::node::cli {
	namespace export -clear *
}

if {[info exists ::nano::node::cli::_using_repl]} {
	::nano::node::cli -import
}