Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch py-b8f575aa23 Through [4888129681] Excluding Merge-Ins
This is equivalent to a diff from 41c19b8112 to 4888129681
|
2023-01-01
| ||
| 23:36 | Merge 8.6 check-in: b85d5ef2c8 user: jan.nijtmans tags: core-8-branch | |
|
2022-12-30
| ||
| 21:05 | New test, io-12.9.strict, for issue report [1bedc53c8cb878f0]. check-in: 703a1a4792 user: pooryorick tags: py-b8f575aa23 | |
| 20:27 | Fix DoReadChars() to correctly discard encoding errors after eofchar has been seen, and add new test... check-in: 4888129681 user: pooryorick tags: py-b8f575aa23 | |
|
2022-12-29
| ||
| 22:59 | Arrange new code in DoReadChars to ensure that final steps are always taken. check-in: 0221bfae40 user: pooryorick tags: py-b8f575aa23 | |
|
2022-12-28
| ||
| 22:58 | merge py-b8f575aa23 after updating fix so that all tests pass. jn: let's wait for more review, si... check-in: 9b6c29fa98 user: pooryorick tags: py-b8f575aa23 | |
| 17:01 | merge 8.7 check-in: 81e8cdd1f0 user: dgp tags: core-8-7-b1-rc | |
| 12:07 | A better fix for [b8f575aa23], as it maintains the expectation that synchronous [read] results in a... check-in: 00995080d6 user: pooryorick tags: py-b8f575aa23 | |
|
2022-12-21
| ||
| 21:43 | Merge 8.7 check-in: fb6e7180d0 user: jan.nijtmans tags: trunk, main | |
| 21:07 | Make two more functions static check-in: 41c19b8112 user: jan.nijtmans tags: core-8-branch | |
| 20:43 | Add 'interp' argument to some arith functions, for better error-handling check-in: ccd603d584 user: jan.nijtmans tags: core-8-branch | |
Changes to generic/tclEncoding.c.
| ︙ | ︙ | |||
2382 2383 2384 2385 2386 2387 2388 | /* * Copy 7bit characters, but skip null-bytes when we are in input * mode, so that they get converted to 0xC080. */ *dst++ = *src++; } else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd) | > > > | > > | 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 |
/*
* Copy 7bit characters, but skip null-bytes when we are in input
* mode, so that they get converted to 0xC080.
*/
*dst++ = *src++;
} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd)
&& (UCHAR(src[1]) == 0x80)
&& (
!(flags & TCL_ENCODING_MODIFIED)
|| ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
))
{
/*
* If in input mode, and -strict is specified: This is an error.
*/
if (flags & TCL_ENCODING_MODIFIED) {
result = TCL_CONVERT_SYNTAX;
break;
}
|
| ︙ | ︙ |
Changes to generic/tclIO.c.
| ︙ | ︙ | |||
6020 6021 6022 6023 6024 6025 6026 |
*/
TclGetString(objPtr);
}
}
if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR)) {
| | > | | | 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 |
*/
TclGetString(objPtr);
}
}
if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR)) {
/* TODO: UpdateInterest not needed here? */
UpdateInterest(chanPtr);
Tcl_SetErrno(EILSEQ);
return -1;
}
/*
* Early out when next read will see eofchar.
*
* NOTE: See DoRead for argument that it's a bug (one we're keeping) to
* have this escape before the one for zero-char read request.
*/
if (GotFlag(statePtr, CHANNEL_STICKY_EOF)) {
SetFlag(statePtr, CHANNEL_EOF);
assert(statePtr->inputEncodingFlags & TCL_ENCODING_END);
assert(!GotFlag(statePtr, CHANNEL_BLOCKED|INPUT_SAW_CR));
/* TODO: UpdateInterest not needed here? */
UpdateInterest(chanPtr);
return 0;
}
/*
* Special handling for zero-char read request.
*/
if (toRead == 0) {
if (GotFlag(statePtr, CHANNEL_EOF)) {
statePtr->inputEncodingFlags |= TCL_ENCODING_START;
}
ResetFlag(statePtr, CHANNEL_BLOCKED|CHANNEL_EOF);
statePtr->inputEncodingFlags &= ~TCL_ENCODING_END;
/* TODO: UpdateInterest not needed here? */
UpdateInterest(chanPtr);
return 0;
}
/*
* This operation should occur at the top of a channel stack.
*/
|
| ︙ | ︙ | |||
6082 6083 6084 6085 6086 6087 6088 |
if (binaryMode) {
copiedNow = ReadBytes(statePtr, objPtr, toRead);
} else {
copiedNow = ReadChars(statePtr, objPtr, toRead, &factor);
}
/*
| | > > > > > > > > > > > > > > > > > > | 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 |
if (binaryMode) {
copiedNow = ReadBytes(statePtr, objPtr, toRead);
} else {
copiedNow = ReadChars(statePtr, objPtr, toRead, &factor);
}
/*
* Recycle current buffer if empty.
*/
bufPtr = statePtr->inQueueHead;
if (IsBufferEmpty(bufPtr)) {
ChannelBuffer *nextPtr = bufPtr->nextPtr;
RecycleBuffer(statePtr, bufPtr, 0);
statePtr->inQueueHead = nextPtr;
if (nextPtr == NULL) {
statePtr->inQueueTail = NULL;
}
}
/*
* If CHANNEL_ENCODING_ERROR and CHANNEL_STICKY_EOF are both set,
* then CHANNEL_ENCODING_ERROR was caused by data that occurred
* after the EOF character was encountered, so it doesn't count as
* a real error.
*/
if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR)
&& !GotFlag(statePtr, CHANNEL_STICKY_EOF)
&& !GotFlag(statePtr, CHANNEL_NONBLOCKING)) {
/* Channel is synchronous. Return an error so that callers
* like [read] can return an error.
*/
Tcl_SetErrno(EILSEQ);
copied = -1;
goto finish;
}
}
if (copiedNow < 0) {
if (GotFlag(statePtr, CHANNEL_EOF)) {
break;
}
if (GotFlag(statePtr, CHANNEL_NONBLOCKING|CHANNEL_BLOCKED)
|
| ︙ | ︙ | |||
6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 |
}
} else {
copied += copiedNow;
toRead -= copiedNow;
}
}
/*
* Failure to fill a channel buffer may have left channel reporting a
* "blocked" state, but so long as we fulfilled the request here, the
* caller does not consider us blocked.
*/
if (toRead == 0) {
| > | 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 |
}
} else {
copied += copiedNow;
toRead -= copiedNow;
}
}
finish:
/*
* Failure to fill a channel buffer may have left channel reporting a
* "blocked" state, but so long as we fulfilled the request here, the
* caller does not consider us blocked.
*/
if (toRead == 0) {
|
| ︙ | ︙ | |||
6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 |
*srcLenPtr = srcLen;
if (srcStart + srcLen == eof) {
/*
* EOF character was seen in EOL translated range. Leave current file
* position pointing at the EOF character, but don't store the EOF
* character in the output string.
*/
SetFlag(statePtr, CHANNEL_EOF | CHANNEL_STICKY_EOF);
statePtr->inputEncodingFlags |= TCL_ENCODING_END;
| > > > | | 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 |
*srcLenPtr = srcLen;
if (srcStart + srcLen == eof) {
/*
* EOF character was seen in EOL translated range. Leave current file
* position pointing at the EOF character, but don't store the EOF
* character in the output string.
*
* If CHANNEL_ENCODING_ERROR is set, it can only be because of data
* encountered after the EOF character, so it is nonsense. Unset it.
*/
SetFlag(statePtr, CHANNEL_EOF | CHANNEL_STICKY_EOF);
statePtr->inputEncodingFlags |= TCL_ENCODING_END;
ResetFlag(statePtr, CHANNEL_BLOCKED|INPUT_SAW_CR|CHANNEL_ENCODING_ERROR);
}
}
/*
*----------------------------------------------------------------------
*
* Tcl_Ungets --
|
| ︙ | ︙ |
Changes to generic/tclIOCmd.c.
| ︙ | ︙ | |||
377 378 379 380 381 382 383 |
Tcl_Obj *const objv[]) /* Argument objects. */
{
Tcl_Channel chan; /* The channel to read from. */
int newline, i; /* Discard newline at end? */
int toRead; /* How many bytes to read? */
int charactersRead; /* How many characters were read? */
int mode; /* Mode in which channel is opened. */
| | | 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 |
Tcl_Obj *const objv[]) /* Argument objects. */
{
Tcl_Channel chan; /* The channel to read from. */
int newline, i; /* Discard newline at end? */
int toRead; /* How many bytes to read? */
int charactersRead; /* How many characters were read? */
int mode; /* Mode in which channel is opened. */
Tcl_Obj *resultPtr, *returnOptsPtr, *chanObjPtr;
if ((objc != 2) && (objc != 3)) {
Interp *iPtr;
argerror:
iPtr = (Interp *) interp;
Tcl_WrongNumArgs(interp, 1, objv, "channelId ?numChars?");
|
| ︙ | ︙ | |||
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 |
*/
if (!TclChanCaughtErrorBypass(interp, chan)) {
Tcl_SetObjResult(interp, Tcl_ObjPrintf(
"error reading \"%s\": %s",
TclGetString(chanObjPtr), Tcl_PosixError(interp)));
}
TclChannelRelease(chan);
Tcl_DecrRefCount(resultPtr);
return TCL_ERROR;
}
/*
* If requested, remove the last newline in the channel if at EOF.
*/
| > > > > | 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 |
*/
if (!TclChanCaughtErrorBypass(interp, chan)) {
Tcl_SetObjResult(interp, Tcl_ObjPrintf(
"error reading \"%s\": %s",
TclGetString(chanObjPtr), Tcl_PosixError(interp)));
}
returnOptsPtr = Tcl_NewDictObj();
Tcl_DictObjPut(NULL, returnOptsPtr, Tcl_NewStringObj("-result", -1)
, resultPtr);
TclChannelRelease(chan);
Tcl_DecrRefCount(resultPtr);
Tcl_SetReturnOptions(interp, returnOptsPtr);
return TCL_ERROR;
}
/*
* If requested, remove the last newline in the channel if at EOF.
*/
|
| ︙ | ︙ |
Changes to tests/io.test.
| ︙ | ︙ | |||
9052 9053 9054 9055 9056 9057 9058 |
fconfigure $f -encoding binary
# \x81 is invalid in utf-8
puts -nonewline $f A\x81
flush $f
seek $f 0
fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
| > | < < | > > | | | > | > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063 9064 9065 9066 9067 9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078 9079 9080 9081 9082 9083 9084 9085 9086 9087 9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098 9099 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 |
fconfigure $f -encoding binary
# \x81 is invalid in utf-8
puts -nonewline $f A\x81
flush $f
seek $f 0
fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
set status [catch {read $f} cres copts]
set d [dict get $copts -result]
binary scan $d H* hd
lappend hd $status $cres
} -cleanup {
close $f
removeFile io-75.6
} -match glob -result {41 1 {error reading "*": illegal byte sequence}}
test io-75.7 {invalid utf-8 encoding eof handling (-strictencoding 1)} -setup {
set fn [makeFile {} io-75.7]
set f [open $fn w+]
fconfigure $f -encoding binary
# \xA1 is invalid in utf-8. -eofchar is not detected, because it comes later.
puts -nonewline $f A\xA1\x1A
flush $f
seek $f 0
fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -strictencoding 1
} -body {
set status [catch {read $f} cres copts]
set d [dict get $copts -result]
binary scan $d H* hd
lappend hd [eof $f]
lappend hd $status
lappend hd $cres
fconfigure $f -encoding iso8859-1
lappend hd [read $f];# We changed encoding, so now we can read the \xA1
close $f
set hd
} -cleanup {
removeFile io-75.7
} -match glob -result {41 0 1 {error reading "*": illegal byte sequence} ¡}
test io-75.8.incomplete {
incomplete uft-8 char after eof char is not an error (-strictencoding 1)
} -setup {
set hd {}
set fn [makeFile {} io-75.8]
set f [open $fn w+]
fconfigure $f -encoding binary
# \x81 is invalid and also incomplete utf-8 data, but because the eof
# character \x1A appears first, it's not an error.
puts -nonewline $f A\x1A\x81
flush $f
seek $f 0
fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -strictencoding 1
} -body {
set d [read $f]
binary scan $d H* hd
lappend hd [eof $f]
# there should be no error on additional reads
lappend hd [read $f]
close $f
set hd
} -cleanup {
removeFile io-75.8
} -result {41 1 {}}
test io-75.8.invalid {invalid utf-8 after eof char is not an error (-strictencoding 1)} -setup {
set res {}
set fn [makeFile {} io-75.8]
set f [open $fn w+]
fconfigure $f -encoding binary
# \xc0\x80 is invalid utf-8 data, but because the eof character \x1A
# appears first, it's not an error.
puts -nonewline $f A\x1a\xc0\x80
flush $f
seek $f 0
fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -strictencoding 1
} -body {
set d [read $f]
foreach char [split $d {}] {
lappend res [format %x [scan $char %c]]
}
lappend res [eof $f]
# there should be no error on additional reads
lappend res [read $f]
close $f
set res
} -cleanup {
removeFile io-75.8
} -result {41 1 {}}
test io-75.9 {unrepresentable character write passes and is replaced by ?} -setup {
set fn [makeFile {} io-75.9]
set f [open $fn w+]
fconfigure $f -encoding iso8859-1 -strictencoding 1
} -body {
catch {puts -nonewline $f "A\u2022"} msg
|
| ︙ | ︙ | |||
9153 9154 9155 9156 9157 9158 9159 |
# In shiftjis, \x81 starts a two-byte sequence.
# But 2nd byte \xFF is not allowed
puts -nonewline $f A\x81\xFFA
flush $f
seek $f 0
fconfigure $f -encoding shiftjis -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
| > | | | | 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 |
# In shiftjis, \x81 starts a two-byte sequence.
# But 2nd byte \xFF is not allowed
puts -nonewline $f A\x81\xFFA
flush $f
seek $f 0
fconfigure $f -encoding shiftjis -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
set status [catch {read $f} cres copts]
set d [dict get $copts -result]
binary scan $d H* hd
lappend hd $status
lappend hd $cres
} -cleanup {
close $f
removeFile io-75.11
} -match glob -result {41 1 {error reading "*": illegal byte sequence}}
test io-75.12 {invalid utf-8 encoding read is ignored} -setup {
set fn [makeFile {} io-75.12]
|
| ︙ | ︙ | |||
9188 9189 9190 9191 9192 9193 9194 |
fconfigure $f -encoding binary
# \x81 is invalid in utf-8
puts -nonewline $f "A\x81"
flush $f
seek $f 0
fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
| > | | | | 9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 |
fconfigure $f -encoding binary
# \x81 is invalid in utf-8
puts -nonewline $f "A\x81"
flush $f
seek $f 0
fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
set status [catch {read $f} cres copts]
set d [dict get $copts -result]
binary scan $d H* hd
lappend hd $status
close $f
lappend hd $cres
} -cleanup {
removeFile io-75.13
} -match glob -result {41 1 {error reading "*": illegal byte sequence}}
# ### ### ### ######### ######### #########
|
| ︙ | ︙ |