Artifact [0ca12aeeb6]

Artifact 0ca12aeeb693c6fdb8ce31d4c5c3dc3c70f35d21:


     1
     2
     3
     4
     5
     6
     7
     8
     9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
    87
    88
    89
    90
    91
    92
    93
    94
    95
    96
    97
    98
    99
   100
   101
   102
   103
   104
   105
   106
   107
   108
   109
   110
   111
   112
   113
   114
   115
   116
   117
   118
   119
   120
   121
   122
   123
   124
   125
   126
   127
   128
   129
   130
   131
   132
   133
   134
   135
   136
   137
   138
   139
   140
   141
   142
   143
   144
   145
   146
   147
   148
   149
   150
   151
   152
   153
   154
   155
   156
   157
   158
   159
   160
   161
   162
   163
   164
   165
   166
   167
   168
   169
   170
   171
   172
   173
   174
   175
   176
   177
   178
   179
   180
   181
   182
   183
   184
   185
   186
   187
   188
   189
   190
   191
   192
   193
   194
   195
   196
   197
   198
   199
   200
   201
   202
   203
   204
   205
   206
   207
   208
   209
   210
   211
   212
   213
   214
   215
   216
   217
   218
   219
   220
   221
   222
   223
   224
   225
   226
   227
   228
   229
   230
   231
   232
   233
   234
   235
   236
   237
   238
   239
   240
   241
   242
   243
   244
   245
   246
   247
   248
   249
   250
   251
   252
   253
   254
   255
   256
   257
   258
   259
   260
   261
   262
   263
   264
   265
   266
   267
   268
   269
   270
   271
   272
   273
   274
   275
   276
   277
   278
   279
   280
   281
   282
   283
   284
   285
   286
   287
   288
   289
   290
   291
   292
   293
   294
   295
   296
   297
   298
   299
   300
   301
   302
   303
   304
   305
   306
   307
   308
   309
   310
   311
   312
   313
   314
   315
   316
   317
   318
   319
   320
   321
   322
   323
   324
   325
   326
   327
   328
   329
   330
   331
   332
   333
   334
   335
   336
   337
   338
   339
   340
   341
   342
   343
   344
   345
   346
   347
   348
   349
   350
   351
   352
   353
   354
   355
   356
   357
   358
   359
   360
   361
   362
   363
   364
   365
   366
   367
   368
   369
   370
   371
   372
   373
   374
   375
   376
   377
   378
   379
   380
   381
   382
   383
   384
   385
   386
   387
   388
   389
   390
   391
   392
   393
   394
   395
   396
   397
   398
   399
   400
   401
   402
   403
   404
   405
   406
   407
   408
   409
   410
   411
   412
   413
   414
   415
   416
   417
   418
   419
   420
   421
   422
   423
   424
   425
   426
   427
   428
   429
   430
   431
   432
   433
   434
   435
   436
   437
   438
   439
   440
   441
   442
   443
   444
   445
   446
   447
   448
   449
   450
   451
   452
   453
   454
   455
   456
   457
   458
   459
   460
   461
   462
   463
   464
   465
   466
   467
   468
   469
   470
   471
   472
   473
   474
   475
   476
   477
   478
   479
   480
   481
   482
   483
   484
   485
   486
   487
   488
   489
   490
   491
   492
   493
   494
   495
   496
   497
   498
   499
   500
   501
   502
   503
   504
   505
   506
   507
   508
   509
   510
   511
   512
   513
   514
   515
   516
   517
   518
   519
   520
   521
   522
   523
   524
   525
   526
   527
   528
   529
   530
   531
   532
   533
   534
   535
   536
   537
   538
   539
   540
   541
   542
   543
   544
   545
   546
   547
   548
   549
   550
   551
   552
   553
   554
   555
   556
   557
   558
   559
   560
   561
   562
   563
   564
   565
   566
   567
   568
   569
   570
   571
   572
   573
   574
   575
   576
   577
   578
   579
   580
   581
   582
   583
   584
   585
   586
   587
   588
   589
   590
   591
   592
   593
   594
   595
   596
   597
   598
   599
   600
   601
   602
   603
   604
   605
   606
   607
   608
   609
   610
   611
   612
   613
   614
   615
   616
   617
   618
   619
   620
   621
   622
   623
   624
   625
   626
   627
   628
   629
   630
   631
   632
   633
   634
   635
   636
   637
   638
   639
   640
   641
   642
   643
   644
   645
   646
   647
   648
   649
   650
   651
   652
   653
   654
   655
   656
   657
   658
   659
   660
   661
   662
   663
   664
   665
   666
   667
   668
   669
   670
   671
   672
   673
   674
   675
   676
   677
   678
   679
   680
   681
   682
   683
   684
   685
   686
   687
   688
   689
   690
   691
   692
   693
   694
   695
   696
   697
   698
   699
   700
   701
   702
   703
   704
   705
   706
   707
   708
   709
   710
   711
   712
   713
   714
   715
   716
   717
   718
   719
   720
   721
   722
   723
   724
   725
   726
   727
   728
   729
   730
   731
   732
   733
   734
   735
   736
   737
   738
   739
   740
   741
   742
   743
   744
   745
   746
   747
   748
   749
   750
   751
   752
   753
   754
   755
   756
   757
   758
   759
   760
   761
   762
   763
   764
   765
   766
   767
   768
   769
   770
   771
   772
   773
   774
   775
   776
   777
   778
   779
   780
   781
   782
   783
   784
   785
   786
   787
   788
   789
   790
   791
   792
   793
   794
   795
   796
   797
   798
   799
   800
   801
   802
   803
   804
   805
   806
   807
   808
   809
   810
   811
   812
   813
   814
   815
   816
   817
   818
   819
   820
   821
   822
   823
   824
   825
   826
   827
   828
   829
   830
   831
   832
   833
   834
   835
   836
   837
   838
   839
   840
   841
   842
   843
   844
   845
   846
   847
   848
   849
   850
   851
   852
   853
   854
   855
   856
   857
   858
   859
   860
   861
   862
   863
   864
   865
   866
   867
   868
   869
   870
   871
   872
   873
   874
   875
   876
   877
   878
   879
   880
   881
   882
   883
   884
   885
   886
   887
   888
   889
   890
   891
   892
   893
   894
   895
   896
   897
   898
   899
   900
   901
   902
   903
   904
   905
   906
   907
   908
   909
   910
   911
   912
   913
   914
   915
   916
   917
   918
   919
   920
   921
   922
   923
   924
   925
   926
   927
   928
   929
   930
   931
   932
   933
   934
   935
   936
   937
   938
   939
   940
   941
   942
   943
   944
   945
   946
   947
   948
   949
   950
   951
   952
   953
   954
   955
   956
   957
   958
   959
   960
   961
   962
   963
   964
   965
   966
   967
   968
   969
   970
   971
   972
   973
   974
   975
   976
   977
   978
   979
   980
   981
   982
   983
   984
   985
   986
   987
   988
   989
   990
   991
   992
   993
   994
   995
   996
   997
   998
   999
  1000
  1001
  1002
  1003
  1004
  1005
  1006
  1007
  1008
  1009
  1010
  1011
  1012
  1013
  1014
  1015
  1016
  1017
  1018
  1019
  1020
  1021
  1022
  1023
  1024
  1025
  1026
  1027
  1028
  1029
  1030
  1031
  1032
  1033
  1034
  1035
  1036
  1037
  1038
  1039
  1040
  1041
  1042
  1043
  1044
  1045
  1046
  1047
  1048
  1049
  1050
  1051
  1052
  1053
  1054
  1055
  1056
  1057
  1058
  1059
  1060
  1061
  1062
  1063
  1064
  1065
  1066
  1067
  1068
  1069
  1070
  1071
  1072
  1073
  1074
  1075
  1076
  1077
  1078
  1079
  1080
  1081
  1082
  1083
  1084
  1085
  1086
  1087
  1088
  1089
  1090
  1091
  1092
  1093
  1094
  1095
  1096
  1097
  1098
#! /usr/bin/env tclsh

package require Tcl 8.6
package require Thread

namespace eval ::pipethread {}

# Default command to clone an interpreter into a new thread
proc ::pipethread::_cloneInterp {thread {namespace "::"}} {
	# Load all currently loaded packages into the new thread
	if {$namespace eq "::"} {
		thread::send $thread [list set auto_path $::auto_path]

		foreach package [package names] {
			set present 0
			catch {
				package present $package
				set present 1
			}

			if {!$present} {
				continue
			}

			thread::send $thread [list package require $package]
		}
	}

	# Create the namespace we are operating on
	thread::send $thread [list namespace eval $namespace ""]

	# Create a valid namespace prefix string
	set namespacePrefix $namespace
	if {$namespacePrefix ne "::"} {
		append namespacePrefix "::"
	}

	# Copy all namespace variables into the new thread
	foreach var [info vars ${namespacePrefix}*] {
		# Skip variables explicitly marked as noclone
		if {[string match "*_NOCLONE" $var]} {
			continue
		}

		if {[array exists $var]} {
			thread::send $thread [list array set $var [array get $var]]
		} elseif {[info exists $var]} {
			thread::send $thread [list set $var [set $var]]
		}
	}

	if {$namespace eq "::"} {
		# Copy all local variables into the new thread as global variables
		# This may not be the best approach -- but it has to go somewhere...
		if {[uplevel 2 {info level}] != 0} {
			foreach var [uplevel 2 {info vars}] {
				if {[uplevel 2 [list array exists $var]]} {
					thread::send $thread [list array set $var [uplevel 2 [list array get $var]]]
				} else {
					thread::send $thread [list set $var [uplevel 2 [list set $var]]]
				}
			}
		}
	}

	# Copy all procedures into the new thread
	foreach proc [info proc ${namespacePrefix}*] {
		## Skip imported procs, hopefully they'll get re-imported
		if {[namespace origin $proc] ne [namespace which $proc]} {
			continue
		}

		set procArgs [list]
		foreach arg [info args $proc] {
			if {[info default $proc $arg argDefault]} {
				lappend procArgs [list $arg $argDefault]
			} else {
				lappend procArgs $arg
			}
		}

		thread::send $thread [list proc $proc $procArgs [info body $proc]]
	}

	# For every child namespace of the current namespace, repeat this whole process
	foreach childNamespace [namespace children $namespace] {
		_cloneInterp $thread $childNamespace
	}

	return
}

proc ::pipethread::_traceInterp {thread} {
	thread::send $thread [list package require Ttrace]
}

# This invokes the closure responsible for cleaning up a thread that has been terminated
# Arguments:
#      count     -- The number of threads that have been created (also the number of times
#                   this closure will be called)
#      vars      -- A dictionary containing the variables to set in the closure
#      closure   -- The actual lambda itself, it will be [apply]'d
#      _watchVar -- The name of the variable being traced... this is not
#                   passed through to the closure but used for internal
#                   purposes
#      thread    -- The thread that has been terminated
#      _watchOp  -- The operation that occured -- this is an implementation detail
proc ::pipethread::_invokeClosure {count vars closure _watchVar thread _watchOp} {
	# Ignore writes to our internal bookkeeping values, they're not
	# relevant to the closure which only [apply]'d when a thread terminates
	if {$thread == "threadToChan" || $thread == "closureCallCount"} {
		return
	}

	# Determine how many times this closure has been called
	incr ${_watchVar}(closureCallCount)

	set closureCallCount [set ${_watchVar}(closureCallCount)]

	# If this call is the final call, set the appropriate variable
	# to pass to the lambda
	if {$closureCallCount == $count} {
		lappend vars finalInvocation true
	} else {
		lappend vars finalInvocation false
	}

	# Set additional variables the lambda expects
	lappend vars thread $thread
	if {![dict exists $vars threadToChan]} {
		lappend vars threadToChan [set ${_watchVar}(threadToChan)]
	}

	# Invoke the lambda
	apply $closure $vars
}

# Debugging log procedure
# Replace as needed
proc ::pipethread::_debugLog err { return; puts stderr "$err" }

# Starts a set of "coprocessing" pipes, similar to UNIX shell
#
# Each block will execute in its own thread with two additional
# channels: inchan and outchan
#
# These channels represent pipes coming from the previous block (inchan)
# or going to the next block (outchan)
#
# Blocks may either be command prefixes or a block of commands
#
# If a block is a command prefix then two values are appended to it before
# being invoked: inchan outchan
#
# If a block is a block of commands then two variables are set: inchan outchan
# before the block is started.
#
# Examples of both:
#     1. Command prefixes:
#          pipethread::pipe firstBlock | secondBlock | thirdBlock abcdef
#
#        This creates three threads and in each thread invokes:
#            a. firstBlock $inchan $outchan
#            b. secondBlock $inchan $outchan
#            c. thirdBlock abcdef $inchan $outchan
#
#     2. Block of commands:
#          pipethread::pipe { puts $outchan ok } | { gets $inchan line; puts $line }
#
#        This creates two threads and in each thread invokes:
#            a. set inchan <inchan>; set outchan <outchan>; puts $outchan ok
#            b. set inchan <inchan>; set outchan <outchan>; gets $inchan line; puts $line
#
# You may use both types together.
#
# Optional arguments:
#      -cloneInterpCommand <commandPrefix>
#           Specify the command to run to clone the interpreter into each
#           new thread.  If specified as the empty string then new
#           interpreters will have no initialization done on them.
#
#           This is a command prefix and the thread ID is added to it.
#
#           There are some magic values for this command prefix:
#                  _DEFAULT: Use the default command prefix
#                  _TTRACE:  Use the Ttrace mechanism
#
#      -inchan <chan>
#           Specify a channel to be used as the input channel to the
#           first block.  If this is not specified then the first block
#           will get "" as its input channel.
#
#      -outchan <chan>
#           Specify a channel to be used as the output channel from the
#           final block.  If this is not specified then the final block
#           will get either "" or a pipe channel as its output channel.
#           The default is to create a pipe channel and anything written
#           to it will be [return]'d by pipechan::pipe, however if -async
#           mode is used (below) then this channel will not be created and
#           the final block will get "" as its output channel unless this
#           option is used.
#
#      -async <cleanup>
#           Specify that:
#               1. pipethread::pipe should not wait in the event loop (by
#                  calling [vwait]); and
#               2. pipethread::pipe should return immediately after starting
#                  the threads and connecting their pipes, the caller is
#                  then responsible for waiting in the event loop so that
#                  thread termination is handled.
#           When all the blocks terminate (and the final block may not always
#           by the last block to terminate) the script <cleanup> is invoked
#           in the global scope
#
#      --
#           Signals the end of arguments
proc ::pipethread::pipe args {
	# Create a unique handle for this series of blocks and pipes
	incr [namespace current]::handles
	set handle [set [namespace current]::handles]

	# Special values for the -cloneInterpCommand option
	set cloneInterpCommandMagic(_DEFAULT) [namespace current]::_cloneInterp
	set cloneInterpCommandMagic(_TTRACE) [namespace current]::_traceInterp

	# Process commandline arguments
	## Set default values
	set cloneInterpCommand _DEFAULT
	set initialThreadReader ""
	set finalThreadWriter ""
	set operateAsync false

	## Collect arguments
	for {set idx 0} {$idx < [llength $args]} {incr idx} {
		set arg [lindex $args $idx]
		switch -glob -- $arg {
			"-cloneInterpCommand" {
				incr idx

				set cloneInterpCommand [lindex $args $idx]
			}
			"-inchan" {
				incr idx

				set initialThreadReader [lindex $args $idx]
			}
			"-outchan" {
				incr idx

				set finalThreadWriter [lindex $args $idx]
			}
			"-async" {
				set operateAsync true

				incr idx

				set onCompleteCleanup [lindex $args $idx]
			}
			"--" {
				incr idx

				break
			}
			"-*" {
				return -code error "Invalid option: $arg, must be one of: -cloneInterpCommand, -inchan, -outchan, -async, or --"
			}
			default {
				break
			}
		}
	}

	## Update the list of arguments to remove the ones we consumed
	set args [lrange $args $idx end]

	# Replace the cloneInterpCommand magic values with actual ones if appropriate
	if {[info exists cloneInterpCommandMagic($cloneInterpCommand)]} {
		set cloneInterpCommand $cloneInterpCommandMagic($cloneInterpCommand)
	}

	# If we are asked to read and write from the same channel (for example
	# a socket) and we have more than one block, we need to create two
	# channels from it since we need to read from the channel in the first block
	# and write to it in the final block but we cannot give the channel to two
	# different threads
	#
	# We create 2 new pipe-pairs and a thread to bridge them and we copy between
	# them in the event loop.
	if {$initialThreadReader eq $finalThreadWriter && $finalThreadWriter ne "" && [lsearch -exact $args "|"] != -1} {
		set copyThreadChannel $initialThreadReader

		set copyThread [thread::create]

		thread::transfer $copyThread $copyThreadChannel
		thread::send $copyThread [list set copyThreadChannel $copyThreadChannel]
		set copyPipe [thread::send $copyThread {
			set readerPipe [chan pipe]
			set writerPipe [chan pipe]

			set readerPipeReader [lindex $readerPipe 0]
			set readerPipeWriter [lindex $readerPipe 1]
			set writerPipeReader [lindex $writerPipe 0]
			set writerPipeWriter [lindex $writerPipe 1]

			foreach fd [list $readerPipeReader $readerPipeWriter $writerPipeReader $writerPipeWriter] {
				fconfigure $fd -translation binary
			}

			thread::detach $writerPipeReader
			thread::detach $readerPipeWriter

			proc copyComplete {var args} {
				incr ::copyComplete

				set sock [set ::$var]

				unset ::$var

				close $sock
			}

			proc terminate {} {
				if {[info exists ::writerPipeWriter]} {
					close $::writerPipeWriter
				}

				if {[info exists ::readerPipeReader]} {
					close $::readerPipeReader
				}

				thread::detach $::copyThreadChannel
			}

			fcopy $copyThreadChannel $writerPipeWriter -command [list copyComplete writerPipeWriter]
			fcopy $readerPipeReader $copyThreadChannel -command [list copyComplete readerPipeReader]

			list $writerPipeReader $readerPipeWriter
		}]

		set copyPipeReader [lindex $copyPipe 0]
		set copyPipeWriter [lindex $copyPipe 1]

		thread::attach $copyPipeReader
		thread::attach $copyPipeWriter

		set initialThreadReader $copyPipeReader
		set finalThreadWriter $copyPipeWriter
	}

	# Process the list of blocks
	set blockThreadList [list]
	set toRun [list]
	set finalBlock false
	for {set idx 0} {$idx < [llength $args]} {incr idx} {
		set start $idx

		set end [lsearch -start $start -exact $args "|"]

		if {$end == -1} {
			set finalBlock true

			set end [llength $args]
		}

		# Determine the command to run, it is between the two pipes.
		set command [lrange $args $start $end-1]

		# Determine if the block is a command prefix or a block of code
		#
		# This is not easy to do authoritatively, we are just faking it
		# we determine that it is a block of code if:
		#      1. It is a single list element long
		#      2. The string representation begins with the list markers '{' and '}'
		# This can fail if you use a command prefix that has special characters in it
		# and accepts no additional parameters.  You can workaround it by supplying a
		# dummy parameter.
		set anonymous false
		if {[llength $command] == 1 && [string index $command 0] == "\{" && [string index $command end] == "\}"} {
			set anonymous true

			set command [lindex $command 0]
		} else {
			switch -- [lindex $command 0] {
				"exec" {
					set command [join [list \
						[list set command $command] \
						{if {$inchan ne ""} { lappend command <@ $inchan } } \
						{if {$outchan ne ""} { lappend command >@ $outchan } } \
						{eval $command} \
					] "\n"]
					set command [list apply [list {inchan outchan} $command]]
				}
				"foreach" {
					set lineVar [lindex $command 1]
					set commandBlock [lindex $command 2]
					set command {
						uplevel #0 [list set inchan $inchan]
						uplevel #0 [list set outchan $outchan]
						uplevel #0 {
							while true {
								gets $inchan @@line@@
								if {[eof $inchan] && $@@line@@ eq ""} {
									break
								}

								@@command@@
							}
						}
					}

					set command [string map [list @@line@@ $lineVar @@command@@ $commandBlock] $command]
					set command [list apply [list {inchan outchan} $command]]
				}
				"lmap" {
					set lineVar [lindex $command 1]
					set commandBlock [lindex $command 2]
					set command {
						uplevel #0 [list set inchan $inchan]
						uplevel #0 [list set outchan $outchan]
						uplevel #0 {
							while true {
								gets $inchan @@line@@
								if {[eof $inchan] && $@@line@@ eq ""} {
									break
								}

								foreach ::pipethread::internal_line_NOCLONE [@@command@@] { puts $outchan $::pipethread::internal_line_NOCLONE }
							}
						}
					}

					set command [string map [list @@line@@ $lineVar @@command@@ $commandBlock] $command]
					set command [list apply [list {inchan outchan} $command]]
				}
			}
		}

		# Create the pipe to link the output of this thread/block
		# to the input of the next thread/block
		if {!$finalBlock || ($finalThreadWriter eq "" && !$operateAsync)} {
			## For most blocks, a standard pipe is used
			## Also for the final block a pipe is created that is
			## tied to a callback to read into a variable unless
			## an output channel is given with -outchan
			##
			## If we are operating asyncronously then we do
			## not need to do this for the final block
			## even if no -outchan was specified since
			## we do not read from it, since we cannot
			## return it anywhere
			set pipe [chan pipe]
		} else {
			set pipe [list "" $finalThreadWriter]
		}

		set pipeReader [lindex $pipe 0]
		set pipeWriter [lindex $pipe 1]

		# If this block needs a new thread, create a new thread and
		# park it in the event loop
		set thread [thread::create]

		# If this is the first block, remember that so we
		# know its threadID for later use
		if {$start == 0} {
			set initialThread $thread
		}

		# Just keep track of all this information so we can
		# start executing code in the threads AFTER we have
		# created all the pipes
		lappend toRun [list thread $thread anonymous $anonymous command $command pipeReader $pipeReader pipeWriter $pipeWriter]
		lappend blockThreadList $thread

		set idx $end
	}

	# Keep track of the threadID for the final block
	set finalThread $thread

	# If we are operating syncronously and we were not given a channel to
	# write the final block to then we created a pipe -- monitor that pipe
	# and accumulate its contents into our return buffer for this series
	# of blocks. 
	if {$finalThreadWriter eq "" && !$operateAsync} {
		set finalThreadReader $pipeReader

		set returnValueVariable "[namespace current]::returnValue_$handle"

		fconfigure $finalThreadReader -blocking 0 -translation binary
		fileevent $finalThreadReader readable "append $returnValueVariable \[read $finalThreadReader\]"
	}

	# This is the name of the array that we will be using to get notified
	# of thread termination either by [vwait]'ing it directly or by
	# [trace add variable] invoking is asyncronously if -async is used
	set threadResultVariable "[namespace current]::resultArray_$handle"

	# Create the list of variables and their values to pass to our thread
	# cleanup closure
	set cleanupClosureVars [list]
	foreach var [list handle operateAsync initialThread finalThread initialThreadReader finalThreadWriter copyThread copyThreadChannel onCompleteCleanup threadResultVariable] {
		if {[info exists $var]} {
			lappend cleanupClosureVars $var [set $var]
		}
	}

	# Our thread cleanup closure
	#
	# This is invoked whenever a thread terminates, local variables are passed in using a dict (_vars),
	# each key is the name of a local variable and the value is the value of that variable.
	set cleanupClosure {_vars {
		foreach {_var _val} $_vars {
			set $_var $_val
		}

		_debugLog "\[$thread\]: <-EXIT-"

		thread::send $thread [list foreach chan [dict get $threadToChan $thread] { catch { close $chan } }]

		if {![info exists copyThread]} {
			if {$thread eq $initialThread} {
				if {$initialThreadReader ni [list "" stdin $finalThreadWriter]} {
					thread::send $thread [list thread::detach $initialThreadReader]

					thread::attach $initialThreadReader
				}
			}

			if {$thread eq $finalThread} {
				if {$finalThreadWriter ni [list "" stdout stderr]} {
					thread::send $thread [list thread::detach $finalThreadWriter]

					thread::attach $finalThreadWriter
				}
			}
		}

		_debugLog "\[$thread\]: <-RELEASE-"

		thread::release -wait $thread

		_debugLog "\[$thread\]: <-DONE-"

		if {$finalInvocation} {
			# Work around a bug in Tcl:
			#    http://core.tcl.tk/thread/tktview/84be1b5a7
			if {$operateAsync} {
				after idle [list unset -nocomplain $threadResultVariable]
			} else {
				unset -nocomplain $threadResultVariable
			}

			if {[info exists copyThread]} {
				thread::send $copyThread [list terminate]

				thread::release -wait $copyThread

				thread::attach $copyThreadChannel
			}

			if {[info exists onCompleteCleanup]} {
				uplevel #0 $onCompleteCleanup
			}
		}
	} }

	lappend cleanupClosure [namespace current]

	# If we are operating asyncronously create the thread result variable and
	# add a trace to it.  We do this before invoking the blocks so that if they
	# terminate we are notified of it even if they do so before we are finished
	# requesting all of them start
	if {$operateAsync} {
		array set $threadResultVariable [list]

		trace add variable $threadResultVariable [list write] [list [namespace current]::_invokeClosure [llength $toRun] $cleanupClosureVars $cleanupClosure]
	}

	# Start executing each block, from first to last
	#
	# We send the script to each thread asyncronously and configure
	# it to notify us via a write to an array with the thread ID as
	# the array key
	for {set idx 0} {$idx < [llength $toRun]} {incr idx} {
		set info [lindex $toRun $idx]

		# Determine the input channel and output channel
		## The input channel is the read side of the pipe
		## from the previous block, if there was no previous
		## block then the input channel is the -inchan value
		## (blank by default)
		if {$idx > 0} {
			set inchan [dict get [lindex $toRun $idx-1] pipeReader]
		} else {
			set inchan $initialThreadReader
		}
		set outchan [dict get $info pipeWriter]

		set thread [dict get $info thread]
		set anonymous [dict get $info anonymous]
		set command [dict get $info command]

		# Transfer channels that this thread needs to use to that
		# thread.
		#
		# Keep track of the channels which the thread needs to close
		# when it terminates as well (threadToChan) because we need
		# to explicitly do this to avoid background errors.
		dict set ${threadResultVariable}(threadToChan) $thread [list]

		if {$inchan ne ""} {
			if {$inchan ne [list stdin]} {
				thread::transfer $thread $inchan

				if {$inchan ne $initialThreadReader || [info exists copyThread]} {
					dict lappend ${threadResultVariable}(threadToChan) $thread $inchan
				}
			}
		}

		if {$outchan ne ""} {
			if {$outchan ni [list stdout stderr $inchan]} {
				thread::transfer $thread $outchan

				if {$outchan ne $finalThreadWriter || [info exists copyThread]} {
					dict lappend ${threadResultVariable}(threadToChan) $thread $outchan
				}
			}
		}

		# Clone the current interpreter into the thread interpreter as
		# requested by the user.
		if {$cloneInterpCommand ne ""} {
			$cloneInterpCommand $thread
		}

		# Update the command to execute with the parameters we have determined added.
		if {!$anonymous} {
			lappend command $inchan $outchan
		} else {
			set command "set inchan \"$inchan\";set outchan \"$outchan\"\n$command"
		}

		# Give the thread a variable that lists all the other threads in the
		thread::send $thread [list namespace eval ::pipethread [list set ::pipethread::blockThreadList $blockThreadList]]

		# Store inchan and outchan in a namespace variable so that we can retrieve it
		thread::send $thread [list namespace eval ::pipethread [list set ::pipethread::inchan_NOCLONE $inchan]]
		thread::send $thread [list namespace eval ::pipethread [list set ::pipethread::outchan_NOCLONE $outchan]]

		# Send the command to the thread interpreter
		_debugLog "\[$thread\]: -EXEC-> [string map [list "\n" "\\n"] $command]"
		thread::send -async $thread $command ${threadResultVariable}($thread)
	}

	# At this point, if we are operating asyncronously then we are done.
	# The event loop will invoke our variable traces when the commands specified
	# in the [thread::send] immediately above completes for each block and we
	# will clean those threads then
	if {$operateAsync} {
		# Return something that could be meaningful, the list of
		# threads involved in this group of blocks
		return $blockThreadList
	}

	# If we are operating syncronously then we need to wait here for the result
	# array to be written to...
	if {!$operateAsync} {
		# Add the complete list of channels for each thread to the closure
		# variable list, since we know them all now
		lappend cleanupClosureVars threadToChan [set ${threadResultVariable}(threadToChan)]
		unset ${threadResultVariable}(threadToChan)

		# Wait in the event loop for our variable to be written to.
		# Wait exactly the same number of times as we created threads
		# since each thread will set the variable when it terminates
		for {set idx 0} {$idx < [llength $toRun]} {incr idx} {
			_debugLog "\[---\]: WAITING"
			vwait $threadResultVariable

			# Determine the thread ID from the array key that has
			# been set
			set thread [lindex [array names $threadResultVariable] 0]
			unset ${threadResultVariable}($thread)

			# If this is the last block to complete, note it to pass
			# to the lambda
			if {$idx == ([llength $toRun] - 1)} {
				set finalInvocation true
			} else {
				set finalInvocation false
			}

			# Construct a local copy the local variables dict
			# for the lambda so we can update it with values
			# that are specific to this invokation
			set varsDict $cleanupClosureVars

			lappend varsDict thread $thread
			lappend varsDict finalInvocation $finalInvocation

			# Call the lambda
			apply $cleanupClosure $varsDict
		}

		# At this point all the threads have been terminated, including any
		# final cleanup code
		#
		# If there is no output channel given to the last block that means
		# a pipe was created and we should consume anything that remains in
		# it and close our side (the other side is closed already, all threads
		# are gone) and accumulate it further into our buffer and return
		# that buffer
		if {$finalThreadWriter eq ""} {
			append $returnValueVariable [read $finalThreadReader]

			close $finalThreadReader

			set returnValue [set $returnValueVariable]

			unset $returnValueVariable

			return $returnValue
		}
	}

	return
}

# Return the inchan/outchan
proc ::pipethread::inchan {} {
	if {![info exists ::pipethread::inchan_NOCLONE]} {
		return stdin
	}

	return $::pipethread::inchan_NOCLONE
}

proc ::pipethread::outchan {} {
	if {![info exists ::pipethread::outchan_NOCLONE]} {
		return stdout
	}

	return $::pipethread::outchan_NOCLONE
}

# Provide an infinite buffer that will read from inchan as it is readable and write to
# outchan as it is writable.
proc ::pipethread::infiniteBuffer {inchan outchan} {
	fconfigure $inchan -blocking false -translation binary
	fconfigure $outchan -blocking false -translation binary

	set ::readComplete false
	set ::copyComplete false

	fileevent $inchan readable [list apply {{inchan outchan} {
		set data [read -nonewline $inchan]

		if {$data ne ""} {
			append ::buffer $data
		} else {
			if {[eof $inchan]} {
				fileevent $inchan readable {}

				set ::readComplete true
			}
		}

		if {[fileevent $outchan writable] eq ""} {
			fileevent $outchan writable [list apply {{outchan} {
				if {[info exists ::buffer]} {
					puts -nonewline $outchan $::buffer
					flush $outchan

					unset ::buffer
				}

				fileevent $outchan writable {}

				if {$::readComplete} {
					set ::copyComplete true
				}
			}} $outchan]
		}
	}} $inchan $outchan]

	vwait ::copyComplete
}

# The found thread may not exist or it may be a different process altogether
proc ::pipethread::getThreadId {relative} {
	set threadIndex [lsearch -exact $::pipethread::blockThreadList [thread::id]]
	set foundThread [lindex $::pipethread::blockThreadList [expr {$threadIndex+$relative}]]

	return $foundThread
}

# XXX: This may go away or be replaced by something better -- its just an experiment
proc ::pipethread::putEof args {
	if {[lindex $args 0] eq "-position"} {
		set position [lindex $args 1]

		set args [lrange $args 2 end]
	} else {
		set position 1
	}

	if {[llength $args] != 1} {
		return -code error "wrong # args: should be putEof ?-position <position>? handle"
	}

	set targetThread [getThreadId $position]

	set handle [lindex $args 0]

	if {[catch {
		thread::send $targetThread [list namespace eval ::pipethread [list lappend ::pipethread::incoming_NOCLONE($handle) [list eof]]]
	}]} {
		return false
	}

	return true
}

proc ::pipethread::put args {
	if {[lindex $args 0] eq "-position"} {
		set position [lindex $args 1]

		set args [lrange $args 2 end]
	} else {
		set position 1
	}

	if {[llength $args] != 2} {
		return -code error "wrong # args: should be put ?-position <position>? handle message"
	}

	set targetThread [getThreadId $position]

	set handle [lindex $args 0]
	set message [lindex $args 1]

	if {[catch {
		thread::send $targetThread [list namespace eval ::pipethread [list lappend ::pipethread::incoming_NOCLONE($handle) [list data $message]]]
	}]} {
		return false
	}

	return true
}

proc ::pipethread::get {handle messageVar} {
	namespace eval ::pipethread {}

	set haveMessage false
	if {[info exists ::pipethread::incoming_NOCLONE($handle)]} {
		if {[llength $::pipethread::incoming_NOCLONE($handle)] > 0} {
			set haveMessage true
		}
	}

	if {!$haveMessage} {
		vwait ::pipethread::incoming_NOCLONE($handle)
	}

	set messageType [lindex $::pipethread::incoming_NOCLONE($handle) 0 0]

	switch -- $messageType {
		"eof" {
			unset ::pipethread::incoming_NOCLONE($handle)

			return false
		}
		"data" {
			upvar $messageVar message

			set message [lindex $::pipethread::incoming_NOCLONE($handle) 0 1]
		}
	}

	set ::pipethread::incoming_NOCLONE($handle) [lrange $::pipethread::incoming_NOCLONE($handle) 1 end]

	return true
}

package provide pipethread 0.1

if {[info exists ::pipethread::demos]} {
package forget pipethread
################################################################################
### DEMOS ######################################################################
################################################################################

# Define some basic procedures
package require Ttrace
ttrace::enable
proc a {inchan outchan} {
	puts "A: STARTING"
	puts $outchan a
	puts $outchan b
	puts $outchan what
	puts $outchan whatwhat
	puts $outchan c
	puts "A: ENDING"
}

proc b {arg inchan outchan} {
	puts "B: STARTING"
	while true {
		gets $inchan line
		if {[eof $inchan] && $line == ""} {
			break
		}

		if {![string match "$arg*" $line]} {
			continue
		}

		puts $outchan $line
		puts "B: SENDING: $line"
	}
	puts "B: ENDING"
}

proc c {inchan outchan} {
	puts "C: STARTING"
	gets $inchan line
	puts "C: Got: $line"
	puts "C: ENDING"
}

# Start doing things
set what what
ttrace::disable

# Output:
#         --- DEMO 1 ---
#         A: STARTING
#         A: ENDING
#         B: STARTING
#         C: STARTING
#         B: SENDING: what
#         B: SENDING: whatwhat
#         B: ENDING
#         C: Got: what
#         C: ENDING
puts "--- DEMO 1 ---"
::pipethread::pipe -cloneInterpCommand _TTRACE a | b $what | c

# Output:
#         --- DEMO 2 ---
#         A: STARTING
#         A: ENDING
#         C: STARTING
#         C: Got: ok
#         C: ENDING
puts "--- DEMO 2 ---"
::pipethread::pipe a | { puts $outchan ok } | c

# Output:
#         --- DEMO 3 ---
#         c.1
#         b.1
#         a.1
#         <blank line>
#
# Note the extra blank line at the bottom
# This is because [puts $outchan $item.1] writes a newline
# and puts [pipethread::pipe] also writes a newline
puts "--- DEMO 3 ---"
puts [pipethread::pipe { puts $outchan "a"; puts $outchan "b"; puts $outchan "c" } | { foreach item [lreverse [read $inchan]] { puts $outchan $item.1 } }]

# Output:
#         --- DEMO 4 ---
#         1:a
#         2:a
#         3:a
#         1:b
#         2:b
#         3:b
#         1:c
#         2:c
#         3:c
puts "--- DEMO 4 ---"
pipethread::pipe {
	puts $outchan "a"
	puts $outchan "b"
	puts $outchan "c"
} | lmap line { list 1:$line 2:$line 3:$line } | foreach line {
	puts $line
}

# Output:
#         --- DEMO 5 ---
#         LINEx1: 4
#         LINEx2: AGAIN: 5
puts "--- DEMO 5 ---"
pipethread::pipe {
	puts $outchan 1
	puts $outchan 2
	puts $outchan 3
	puts $outchan 4
	puts $outchan 5
} | {
	pipethread::pipe -inchan $inchan -outchan $outchan {
		gets $inchan ignore
		gets $inchan accept
		puts $outchan $accept
		gets $inchan ignore
		gets $inchan accept
		puts $outchan $accept
	} | {
		gets $inchan ignore
		gets $inchan accept
		puts $outchan $accept
	}

	pipethread::pipe -inchan $inchan -outchan $outchan {
		gets $inchan line
		puts $outchan "AGAIN: $line"
	}
} | {
	gets $inchan line
	puts "LINEx1: $line"

	gets $inchan line
	puts "LINEx2: $line"
}

# Experiment
#         --- DEMO 6 ---
#         HELLO
puts "--- DEMO 6 ---"
pipethread::pipe -outchan stdout {
	pipethread::put default "HELLO"
	pipethread::putEof default
	while true {
		if {![pipethread::put default "WORLD"]} {
			break
		}
	}
} | {
	pipethread::get default line
	puts $outchan $line
}

# Overloaded exec demo
# Output:
#         --- DEMO 7 ---
#         <pwd>
#         <hostname>
puts "--- DEMO 7 ---"
pipethread::pipe -outchan stdout {
	puts $outchan [info hostname]
	puts $outchan [pwd]
} | exec tac

# Socket demo: Creates a set of threads for every socket created
# When a client connects to the socket the conversation is as follows:
#         [file9] HELLO !  You are 1 !
#         <wait for a line of input>
#         [file9] GOOD TO KNOW: <the input from above> from 127.0.0.1/47966 [file10<->file13]
#         <connection closed>
#
# and the output looks like:
#         --- DEMO SOCKET ---
#         CLOSING SOCK sock7fa6bc03b630
puts "--- DEMO SOCKET ---"
proc handleConnection {sock addr port} {
	incr ::connectionNumber

	pipethread::pipe -async "puts stderr \"CLOSING SOCK $sock\"; close $sock" -inchan $sock -outchan $sock {
		puts $outchan "HELLO !  You are $::connectionNumber !"
		flush $outchan
		gets $inchan line
		puts $outchan "GOOD TO KNOW: $line from $addr/$port \[$inchan<->$outchan\]"
	} | {
		while true {
			gets $inchan line
			if {[eof $inchan] && $line eq ""} {
				break
			}

			puts $outchan "\[$outchan\] $line"
			flush $outchan
		}
	}
}

proc handleConnectionFake {sock addr port} {
	after 1 [list handleConnection $sock $addr $port]
}

socket -server handleConnectionFake 8410

vwait forever
}