Check-in [3d6b421c61]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Bugfixes to the earlier fix of bug cb0373bb33, which broke HTTP/1.0 transactions in which the server indicates neither a Content-Length nor that it will close the socket on completion ("Connection: close"). The HTTP/1.1 rule is that the response header "Connection", if absent, must default to "keep-alive"; but this rule does not apply to HTTP/1.0. Add test http11-3.4 and bump version to 2.9.4.
Timelines: family | ancestors | descendants | both | http-bug-cb0373bb33-again-for-8-6
Files: files | file ages | folders
SHA3-256: 3d6b421c617116541e7f6b056380dcebd5f0ce2c7f50ec68e30b8047cd48c5b8
User & Date: kjnash 2020-08-12 09:20:56.928
Context
2020-08-22
04:09
Merge 8.6 check-in: cc67744e6c user: kjnash tags: http-bug-cb0373bb33-again-for-8-6
2020-08-12
09:20
Bugfixes to the earlier fix of bug cb0373bb33, which broke HTTP/1.0 transactions in which the server... check-in: 3d6b421c61 user: kjnash tags: http-bug-cb0373bb33-again-for-8-6
03:46
Create new branch named "http-bug-cb0373bb33-again-for-8-6" check-in: 64903a675a user: kjnash tags: http-bug-cb0373bb33-again-for-8-6
Changes
Unified Diff Ignore Whitespace Patch
Changes to library/http/http.tcl.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# http.tcl --
#
#	Client-side HTTP for GET, POST, and HEAD commands. These routines can
#	be used in untrusted code that uses the Safesock security policy.
#	These procedures use a callback interface to avoid using vwait, which
#	is not defined in the safe base.
#
# See the file "license.terms" for information on usage and redistribution of
# this file, and for a DISCLAIMER OF ALL WARRANTIES.

package require Tcl 8.6-
# Keep this in sync with pkgIndex.tcl and with the install directories in
# Makefiles
package provide http 2.9.3

namespace eval http {
    # Allow resourcing to not clobber existing data

    variable http
    if {![info exists http]} {
	array set http {













|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# http.tcl --
#
#	Client-side HTTP for GET, POST, and HEAD commands. These routines can
#	be used in untrusted code that uses the Safesock security policy.
#	These procedures use a callback interface to avoid using vwait, which
#	is not defined in the safe base.
#
# See the file "license.terms" for information on usage and redistribution of
# this file, and for a DISCLAIMER OF ALL WARRANTIES.

package require Tcl 8.6-
# Keep this in sync with pkgIndex.tcl and with the install directories in
# Makefiles
package provide http 2.9.4

namespace eval http {
    # Allow resourcing to not clobber existing data

    variable http
    if {![info exists http]} {
	array set http {
961
962
963
964
965
966
967












968
969
970
971
972
973
974
	    # There is a small risk of a race against server timeout.
	    set state(-pipeline) 0
	}
    } else {
	# It's a GET or HEAD.
	set state(-pipeline) $http(-pipeline)
    }













    # See if we are supposed to use a previously opened channel.
    # - In principle, ANY call to http::geturl could use a previously opened
    #   channel if it is available - the "Connection: keep-alive" header is a
    #   request to leave the channel open AFTER completion of this call.
    # - In fact, we try to use an existing channel only if -keepalive 1 -- this
    #   means that at most one channel is left open for each value of







>
>
>
>
>
>
>
>
>
>
>
>







961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
	    # There is a small risk of a race against server timeout.
	    set state(-pipeline) 0
	}
    } else {
	# It's a GET or HEAD.
	set state(-pipeline) $http(-pipeline)
    }

    # We cannot handle chunked encodings with -handler, so force HTTP/1.0
    # until we can manage this.
    if {[info exists state(-handler)]} {
	set state(-protocol) 1.0
    }

    # RFC 7320 A.1 - HTTP/1.0 Keep-Alive is problematic. We do not support it.
    if {$state(-protocol) eq "1.0"} {
	set state(connection) close
	set state(-keepalive) 0
    }

    # See if we are supposed to use a previously opened channel.
    # - In principle, ANY call to http::geturl could use a previously opened
    #   channel if it is available - the "Connection: keep-alive" header is a
    #   request to leave the channel open AFTER completion of this call.
    # - In fact, we try to use an existing channel only if -keepalive 1 -- this
    #   means that at most one channel is left open for each value of
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371






1372
1373
1374


1375
1376
1377
1378
1379
1380
1381
	# work properly.
	fconfigure $state(-querychannel) -blocking 1 -translation binary
	set contDone 0
    }
    if {[info exists state(-method)] && ($state(-method) ne "")} {
	set how $state(-method)
    }
    # We cannot handle chunked encodings with -handler, so force HTTP/1.0
    # until we can manage this.
    if {[info exists state(-handler)]} {
	set state(-protocol) 1.0
    }
    set accept_types_seen 0

    Log ^B$tk begin sending request - token $token

    if {[catch {
	set state(method) $how
	puts $sock "$how $srvurl HTTP/$state(-protocol)"
	if {[dict exists $state(-headers) Host]} {
	    # Allow Host spoofing. [Bug 928154]
	    puts $sock "Host: [dict get $state(-headers) Host]"
	} elseif {$port == $defport} {
	    # Don't add port in this case, to handle broken servers. [Bug
	    # #504508]
	    puts $sock "Host: $host"
	} else {
	    puts $sock "Host: $host:$port"
	}
	puts $sock "User-Agent: $http(-useragent)"
	if {($state(-protocol) >= 1.0) && $state(-keepalive)} {
	    # Send this header, because a 1.1 server is not compelled to treat
	    # this as the default.
	    puts $sock "Connection: keep-alive"
	}
	if {($state(-protocol) > 1.0) && !$state(-keepalive)} {
	    puts $sock "Connection: close" ;# RFC2616 sec 8.1.2.1
	}






	if {[info exists phost] && ($phost ne "") && $state(-keepalive)} {
	    puts $sock "Proxy-Connection: Keep-Alive"
	}


	set accept_encoding_seen 0
	set content_type_seen 0
	dict for {key value} $state(-headers) {
	    set value [string map [list \n "" \r ""] $value]
	    set key [string map {" " -} [string trim $key]]
	    if {[string equal -nocase $key "host"]} {
		continue







<
<
<
<
<


















|







>
>
>
>
>
>
|
|

>
>







1346
1347
1348
1349
1350
1351
1352





1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
	# work properly.
	fconfigure $state(-querychannel) -blocking 1 -translation binary
	set contDone 0
    }
    if {[info exists state(-method)] && ($state(-method) ne "")} {
	set how $state(-method)
    }





    set accept_types_seen 0

    Log ^B$tk begin sending request - token $token

    if {[catch {
	set state(method) $how
	puts $sock "$how $srvurl HTTP/$state(-protocol)"
	if {[dict exists $state(-headers) Host]} {
	    # Allow Host spoofing. [Bug 928154]
	    puts $sock "Host: [dict get $state(-headers) Host]"
	} elseif {$port == $defport} {
	    # Don't add port in this case, to handle broken servers. [Bug
	    # #504508]
	    puts $sock "Host: $host"
	} else {
	    puts $sock "Host: $host:$port"
	}
	puts $sock "User-Agent: $http(-useragent)"
	if {($state(-protocol) > 1.0) && $state(-keepalive)} {
	    # Send this header, because a 1.1 server is not compelled to treat
	    # this as the default.
	    puts $sock "Connection: keep-alive"
	}
	if {($state(-protocol) > 1.0) && !$state(-keepalive)} {
	    puts $sock "Connection: close" ;# RFC2616 sec 8.1.2.1
	}
	if {($state(-protocol) < 1.1)} {
	    # RFC7230 A.1
	    # Some server implementations of HTTP/1.0 have a faulty
	    # implementation of RFC 2068 Keep-Alive.
	    # Don't leave this to chance.
	    # For HTTP/1.0 we have already "set state(connection) close"
	    # and "state(-keepalive) 0".
	    puts $sock "Connection: close"
	}
	# RFC7230 A.1 - "clients are encouraged not to send the
	# Proxy-Connection header field in any requests"
	set accept_encoding_seen 0
	set content_type_seen 0
	dict for {key value} $state(-headers) {
	    set value [string map [list \n "" \r ""] $value]
	    set key [string map {" " -} [string trim $key]]
	    if {[string equal -nocase $key "host"]} {
		continue
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713

2714
2715
2716
2717
2718
2719
2720
			    if {$tmpHeader in {close keep-alive}} {
				# The common cases, continue.
			    } elseif {[string first , $tmpHeader] == -1} {
				# Not a comma-separated list, not "close",
				# therefore "keep-alive".
				set tmpHeader keep-alive
			    } else {
				set tmpHeader keep-alive
				set tmpCsl [split $tmpHeader ,]
				# Optional whitespace either side of separator.
				foreach el $tmpCsl {
				    if {[string trim $el] eq {close}} {
					set tmpHeader close
					break
				    }
			        }

			    }
			    set state(connection) $tmpHeader
			}
		    }
		    lappend state(meta) $key [string trim $value]
		}
	    }







|




|



>







2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
			    if {$tmpHeader in {close keep-alive}} {
				# The common cases, continue.
			    } elseif {[string first , $tmpHeader] == -1} {
				# Not a comma-separated list, not "close",
				# therefore "keep-alive".
				set tmpHeader keep-alive
			    } else {
				set tmpResult keep-alive
				set tmpCsl [split $tmpHeader ,]
				# Optional whitespace either side of separator.
				foreach el $tmpCsl {
				    if {[string trim $el] eq {close}} {
					set tmpResult close
					break
				    }
			        }
				set tmpHeader $tmpResult
			    }
			    set state(connection) $tmpHeader
			}
		    }
		    lappend state(meta) $key [string trim $value]
		}
	    }
Changes to library/http/pkgIndex.tcl.
1
2
if {![package vsatisfies [package provide Tcl] 8.6-]} {return}
package ifneeded http 2.9.3 [list tclPkgSetup $dir http 2.9.3 {{http.tcl source {::http::config ::http::formatQuery ::http::geturl ::http::reset ::http::wait ::http::register ::http::unregister ::http::mapReply}}}]

|
1
2
if {![package vsatisfies [package provide Tcl] 8.6-]} {return}
package ifneeded http 2.9.4 [list tclPkgSetup $dir http 2.9.4 {{http.tcl source {::http::config ::http::formatQuery ::http::geturl ::http::reset ::http::wait ::http::register ::http::unregister ::http::mapReply}}}]
Changes to tests/http11.test.
622
623
624
625
626
627
628



























629
630
631
632
633
634
635
        [expr {[file size testdoc.html]-[string length $testdata]}]
} -cleanup {
    http::cleanup $tok
    unset -nocomplain testdata
    halt_httpd
} -result {ok {HTTP/1.0 200 OK} ok close {} {} 0}




























test http11-4.0 "normal post request" -setup {
    variable httpd [create_httpd]
} -body {
    set query [http::formatQuery q 1 z 2]
    set tok [http::geturl http://localhost:$httpd_port/testdoc.html \
                 -query $query -timeout 10000]
    http::wait $tok







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
        [expr {[file size testdoc.html]-[string length $testdata]}]
} -cleanup {
    http::cleanup $tok
    unset -nocomplain testdata
    halt_httpd
} -result {ok {HTTP/1.0 200 OK} ok close {} {} 0}

# http11-3.4
# This test is a blatant attempt to confuse the client by instructing the server
# to send neither "Connection: close" nor "Content-Length" when in non-chunked
# mode.
# The client has no way to know the response-body is complete unless the
# server signals this by closing the connection.
# In an HTTP/1.1 response the absence of "Connection: close" means
# "Connection: keep-alive", i.e. the server will keep the connection
# open.  In HTTP/1.0 this is not the case, and this is a test that
# the Tcl client assumes "Connection: close" by default in HTTP/1.0.
test http11-3.4 "-handler,close,identity; HTTP/1.0 server does not send Connection: close header or Content-Length" -setup {
    variable httpd [create_httpd]
    set testdata ""
} -body {
    set tok [http::geturl http://localhost:$httpd_port/testdoc.html?close=1&nosendclose=any \
                 -timeout 10000 -handler [namespace code [list handler testdata]]]
    http::wait $tok
    list [http::status $tok] [http::code $tok] [check_crc $tok $testdata]\
        [meta $tok connection] [meta $tok content-encoding] \
        [meta $tok transfer-encoding] \
        [expr {[file size testdoc.html]-[string length $testdata]}]
} -cleanup {
    http::cleanup $tok
    unset -nocomplain testdata
    halt_httpd
} -result {ok {HTTP/1.0 200 OK} ok {} {} {} 0}

test http11-4.0 "normal post request" -setup {
    variable httpd [create_httpd]
} -body {
    set query [http::formatQuery q 1 z 2]
    set tok [http::geturl http://localhost:$httpd_port/testdoc.html \
                 -query $query -timeout 10000]
    http::wait $tok
Changes to tests/httpd11.tcl.
166
167
168
169
170
171
172

173
174
175

176
177
178
179



180
181
182
183
184
185
186
187
188
189
190
191
192
193
194




195
196
197
198
199
200
201
		}
	    }
            set transfer chunked
        } else {
            set close 1
        }


        foreach pair [split $query &] {
            if {[scan $pair {%[^=]=%s} key val] != 2} {set val ""}
            switch -exact -- $key {

                close        {set close 1 ; set transfer 0}
                transfer     {set transfer $val}
                content-type {set type $val}
            }



        }

        chan configure $chan -buffering line -encoding iso8859-1 -translation crlf
        Puts $chan "$protocol $code"
        Puts $chan "content-type: $type"
        Puts $chan [format "x-crc32: %08x" [zlib crc32 $data]]
        if {$req eq "POST"} {
            Puts $chan [format "x-query-length: %d" [string length $query]]
        }
        if {$close} {
            Puts $chan "connection: close"
        }
	Puts $chan "x-requested-encodings: [dict get? $meta accept-encoding]"
        if {$encoding eq "identity"} {
            Puts $chan "content-length: [string length $data]"




        } else {
            Puts $chan "content-encoding: $encoding"
        }
        if {$transfer eq "chunked"} {
            Puts $chan "transfer-encoding: chunked"
        }
        puts $chan ""







>



>




>
>
>









|



|

>
>
>
>







166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
		}
	    }
            set transfer chunked
        } else {
            set close 1
        }

        set nosendclose 0
        foreach pair [split $query &] {
            if {[scan $pair {%[^=]=%s} key val] != 2} {set val ""}
            switch -exact -- $key {
                nosendclose  {set nosendclose 1}
                close        {set close 1 ; set transfer 0}
                transfer     {set transfer $val}
                content-type {set type $val}
            }
        }
        if {$protocol eq "HTTP/1.1"} {
            set nosendclose 0
        }

        chan configure $chan -buffering line -encoding iso8859-1 -translation crlf
        Puts $chan "$protocol $code"
        Puts $chan "content-type: $type"
        Puts $chan [format "x-crc32: %08x" [zlib crc32 $data]]
        if {$req eq "POST"} {
            Puts $chan [format "x-query-length: %d" [string length $query]]
        }
        if {$close && (!$nosendclose)} {
            Puts $chan "connection: close"
        }
	Puts $chan "x-requested-encodings: [dict get? $meta accept-encoding]"
        if {$encoding eq "identity" && (!$nosendclose)} {
            Puts $chan "content-length: [string length $data]"
        } elseif {$encoding eq "identity"} {
            # This is a blatant attempt to confuse the client by sending neither
            # "Connection: close" nor "Content-Length" when in non-chunked mode.
            # See test http11-3.4.
        } else {
            Puts $chan "content-encoding: $encoding"
        }
        if {$transfer eq "chunked"} {
            Puts $chan "transfer-encoding: chunked"
        }
        puts $chan ""
Changes to unix/Makefile.in.
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
	    $(INSTALL_DATA) $$i "$(SCRIPT_INSTALL_DIR)"; \
	    done;
	@echo "Installing package http1.0 files to $(SCRIPT_INSTALL_DIR)/http1.0/";
	@for i in $(TOP_DIR)/library/http1.0/*.tcl ; \
	    do \
	    $(INSTALL_DATA) $$i "$(SCRIPT_INSTALL_DIR)/http1.0"; \
	    done;
	@echo "Installing package http 2.9.3 as a Tcl Module";
	@$(INSTALL_DATA) $(TOP_DIR)/library/http/http.tcl "$(MODULE_INSTALL_DIR)/8.6/http-2.9.3.tm";
	@echo "Installing package opt0.4 files to $(SCRIPT_INSTALL_DIR)/opt0.4/";
	@for i in $(TOP_DIR)/library/opt/*.tcl ; \
	    do \
	    $(INSTALL_DATA) $$i "$(SCRIPT_INSTALL_DIR)/opt0.4"; \
	    done;
	@echo "Installing package msgcat 1.6.1 as a Tcl Module";
	@$(INSTALL_DATA) $(TOP_DIR)/library/msgcat/msgcat.tcl "$(MODULE_INSTALL_DIR)/8.5/msgcat-1.6.1.tm";







|
|







940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
	    $(INSTALL_DATA) $$i "$(SCRIPT_INSTALL_DIR)"; \
	    done;
	@echo "Installing package http1.0 files to $(SCRIPT_INSTALL_DIR)/http1.0/";
	@for i in $(TOP_DIR)/library/http1.0/*.tcl ; \
	    do \
	    $(INSTALL_DATA) $$i "$(SCRIPT_INSTALL_DIR)/http1.0"; \
	    done;
	@echo "Installing package http 2.9.4 as a Tcl Module";
	@$(INSTALL_DATA) $(TOP_DIR)/library/http/http.tcl "$(MODULE_INSTALL_DIR)/8.6/http-2.9.4.tm";
	@echo "Installing package opt0.4 files to $(SCRIPT_INSTALL_DIR)/opt0.4/";
	@for i in $(TOP_DIR)/library/opt/*.tcl ; \
	    do \
	    $(INSTALL_DATA) $$i "$(SCRIPT_INSTALL_DIR)/opt0.4"; \
	    done;
	@echo "Installing package msgcat 1.6.1 as a Tcl Module";
	@$(INSTALL_DATA) $(TOP_DIR)/library/msgcat/msgcat.tcl "$(MODULE_INSTALL_DIR)/8.5/msgcat-1.6.1.tm";
Changes to win/Makefile.in.
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
	    $(COPY) "$$i" "$(SCRIPT_INSTALL_DIR)"; \
	    done;
	@echo "Installing library http1.0 directory";
	@for j in $(ROOT_DIR)/library/http1.0/*.tcl; \
	    do \
	    $(COPY) "$$j" "$(SCRIPT_INSTALL_DIR)/http1.0"; \
	    done;
	@echo "Installing package http 2.9.3 as a Tcl Module";
	@$(COPY) $(ROOT_DIR)/library/http/http.tcl "$(MODULE_INSTALL_DIR)/8.6/http-2.9.3.tm";
	@echo "Installing library opt0.4 directory";
	@for j in $(ROOT_DIR)/library/opt/*.tcl; \
	    do \
	    $(COPY) "$$j" "$(SCRIPT_INSTALL_DIR)/opt0.4"; \
	    done;
	@echo "Installing package msgcat 1.6.1 as a Tcl Module";
	@$(COPY) $(ROOT_DIR)/library/msgcat/msgcat.tcl "$(MODULE_INSTALL_DIR)/8.5/msgcat-1.6.1.tm";







|
|







715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
	    $(COPY) "$$i" "$(SCRIPT_INSTALL_DIR)"; \
	    done;
	@echo "Installing library http1.0 directory";
	@for j in $(ROOT_DIR)/library/http1.0/*.tcl; \
	    do \
	    $(COPY) "$$j" "$(SCRIPT_INSTALL_DIR)/http1.0"; \
	    done;
	@echo "Installing package http 2.9.4 as a Tcl Module";
	@$(COPY) $(ROOT_DIR)/library/http/http.tcl "$(MODULE_INSTALL_DIR)/8.6/http-2.9.4.tm";
	@echo "Installing library opt0.4 directory";
	@for j in $(ROOT_DIR)/library/opt/*.tcl; \
	    do \
	    $(COPY) "$$j" "$(SCRIPT_INSTALL_DIR)/opt0.4"; \
	    done;
	@echo "Installing package msgcat 1.6.1 as a Tcl Module";
	@$(COPY) $(ROOT_DIR)/library/msgcat/msgcat.tcl "$(MODULE_INSTALL_DIR)/8.5/msgcat-1.6.1.tm";