Changes On Branch 4888129681d26baf
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch py-b8f575aa23 Through [4888129681] Excluding Merge-Ins

This is equivalent to a diff from 41c19b8112 to 4888129681

2023-01-01
23:36
Merge 8.6 check-in: b85d5ef2c8 user: jan.nijtmans tags: core-8-branch
2022-12-30
21:05
New test, io-12.9.strict, for issue report [1bedc53c8cb878f0]. check-in: 703a1a4792 user: pooryorick tags: py-b8f575aa23
20:27
Fix DoReadChars() to correctly discard encoding errors after eofchar has been seen, and add new test... check-in: 4888129681 user: pooryorick tags: py-b8f575aa23
2022-12-29
22:59
Arrange new code in DoReadChars to ensure that final steps are always taken. check-in: 0221bfae40 user: pooryorick tags: py-b8f575aa23
2022-12-28
22:58
merge py-b8f575aa23 after updating fix so that all tests pass. jn: let's wait for more review, si... check-in: 9b6c29fa98 user: pooryorick tags: py-b8f575aa23
17:01
merge 8.7 check-in: 81e8cdd1f0 user: dgp tags: core-8-7-b1-rc
12:07
A better fix for [b8f575aa23], as it maintains the expectation that synchronous [read] results in a... check-in: 00995080d6 user: pooryorick tags: py-b8f575aa23
2022-12-21
21:43
Merge 8.7 check-in: fb6e7180d0 user: jan.nijtmans tags: trunk, main
21:07
Make two more functions static check-in: 41c19b8112 user: jan.nijtmans tags: core-8-branch
20:43
Add 'interp' argument to some arith functions, for better error-handling check-in: ccd603d584 user: jan.nijtmans tags: core-8-branch

Changes to generic/tclEncoding.c.
2382
2383
2384
2385
2386
2387
2388



2389



2390
2391
2392
2393
2394
2395
2396
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391

2392
2393
2394
2395
2396
2397
2398
2399
2400
2401







+
+
+
-
+
+
+







	    /*
	     * Copy 7bit characters, but skip null-bytes when we are in input
	     * mode, so that they get converted to 0xC080.
	     */

	    *dst++ = *src++;
	} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd)
	    && (UCHAR(src[1]) == 0x80)
	    && (
		!(flags & TCL_ENCODING_MODIFIED)
		&& (UCHAR(src[1]) == 0x80) && (!(flags & TCL_ENCODING_MODIFIED) || ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT))) {
		|| ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
	    ))
	{
	    /*
	     * If in input mode, and -strict is specified: This is an error.
	     */
	    if (flags & TCL_ENCODING_MODIFIED) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
Changes to generic/tclIO.c.
6020
6021
6022
6023
6024
6025
6026
6027

6028

6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044

6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058

6059
6060
6061
6062
6063
6064
6065
6020
6021
6022
6023
6024
6025
6026

6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044

6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058

6059
6060
6061
6062
6063
6064
6065
6066







-
+

+















-
+













-
+







	     */

	    TclGetString(objPtr);
	}
    }

    if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR)) {
	/* TODO: We don't need this call? */
	/* TODO: UpdateInterest not needed here? */
	UpdateInterest(chanPtr);

	Tcl_SetErrno(EILSEQ);
	return -1;
    }
    /*
     * Early out when next read will see eofchar.
     *
     * NOTE: See DoRead for argument that it's a bug (one we're keeping) to
     * have this escape before the one for zero-char read request.
     */

    if (GotFlag(statePtr, CHANNEL_STICKY_EOF)) {
	SetFlag(statePtr, CHANNEL_EOF);
	assert(statePtr->inputEncodingFlags & TCL_ENCODING_END);
	assert(!GotFlag(statePtr, CHANNEL_BLOCKED|INPUT_SAW_CR));

	/* TODO: We don't need this call? */
	/* TODO: UpdateInterest not needed here? */
	UpdateInterest(chanPtr);
	return 0;
    }

    /*
     * Special handling for zero-char read request.
     */
    if (toRead == 0) {
	if (GotFlag(statePtr, CHANNEL_EOF)) {
	    statePtr->inputEncodingFlags |= TCL_ENCODING_START;
	}
	ResetFlag(statePtr, CHANNEL_BLOCKED|CHANNEL_EOF);
	statePtr->inputEncodingFlags &= ~TCL_ENCODING_END;
	/* TODO: We don't need this call? */
	/* TODO: UpdateInterest not needed here? */
	UpdateInterest(chanPtr);
	return 0;
    }

    /*
     * This operation should occur at the top of a channel stack.
     */
6082
6083
6084
6085
6086
6087
6088
6089

6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101


















6102
6103
6104
6105
6106
6107
6108
6083
6084
6085
6086
6087
6088
6089

6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127







-
+












+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







	    if (binaryMode) {
		copiedNow = ReadBytes(statePtr, objPtr, toRead);
	    } else {
		copiedNow = ReadChars(statePtr, objPtr, toRead, &factor);
	    }

	    /*
	     * If the current buffer is empty recycle it.
	     * Recycle current buffer if empty.
	     */

	    bufPtr = statePtr->inQueueHead;
	    if (IsBufferEmpty(bufPtr)) {
		ChannelBuffer *nextPtr = bufPtr->nextPtr;

		RecycleBuffer(statePtr, bufPtr, 0);
		statePtr->inQueueHead = nextPtr;
		if (nextPtr == NULL) {
		    statePtr->inQueueTail = NULL;
		}
	    }

	    /*
	     * If CHANNEL_ENCODING_ERROR and CHANNEL_STICKY_EOF are both set,
	     * then CHANNEL_ENCODING_ERROR was caused by data that occurred
	     * after the EOF character was encountered, so it doesn't count as
	     * a real error.
	     */

	    if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR)
		    && !GotFlag(statePtr, CHANNEL_STICKY_EOF)
		    && !GotFlag(statePtr, CHANNEL_NONBLOCKING)) {
		/* Channel is synchronous.  Return an error so that callers
		 * like [read] can return an error.
		*/
		Tcl_SetErrno(EILSEQ);
		copied = -1;
		goto finish;
	    }
	}

	if (copiedNow < 0) {
	    if (GotFlag(statePtr, CHANNEL_EOF)) {
		break;
	    }
	    if (GotFlag(statePtr, CHANNEL_NONBLOCKING|CHANNEL_BLOCKED)
6123
6124
6125
6126
6127
6128
6129

6130
6131
6132
6133
6134
6135
6136
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156







+







	    }
	} else {
	    copied += copiedNow;
	    toRead -= copiedNow;
	}
    }

finish:
    /*
     * Failure to fill a channel buffer may have left channel reporting a
     * "blocked" state, but so long as we fulfilled the request here, the
     * caller does not consider us blocked.
     */

    if (toRead == 0) {
6801
6802
6803
6804
6805
6806
6807



6808
6809
6810
6811
6812

6813
6814
6815
6816
6817
6818
6819
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834

6835
6836
6837
6838
6839
6840
6841
6842







+
+
+




-
+







    *srcLenPtr = srcLen;

    if (srcStart + srcLen == eof) {
	/*
	 * EOF character was seen in EOL translated range. Leave current file
	 * position pointing at the EOF character, but don't store the EOF
	 * character in the output string.
	 *
	 * If CHANNEL_ENCODING_ERROR is set, it can only be because of data
	 * encountered after the EOF character, so it is nonsense.  Unset it.
	 */

	SetFlag(statePtr, CHANNEL_EOF | CHANNEL_STICKY_EOF);
	statePtr->inputEncodingFlags |= TCL_ENCODING_END;
	ResetFlag(statePtr, CHANNEL_BLOCKED|INPUT_SAW_CR);
	ResetFlag(statePtr, CHANNEL_BLOCKED|INPUT_SAW_CR|CHANNEL_ENCODING_ERROR);
    }
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_Ungets --
Changes to generic/tclIOCmd.c.
377
378
379
380
381
382
383
384

385
386
387
388
389
390
391
377
378
379
380
381
382
383

384
385
386
387
388
389
390
391







-
+







    Tcl_Obj *const objv[])	/* Argument objects. */
{
    Tcl_Channel chan;		/* The channel to read from. */
    int newline, i;		/* Discard newline at end? */
    int toRead;			/* How many bytes to read? */
    int charactersRead;		/* How many characters were read? */
    int mode;			/* Mode in which channel is opened. */
    Tcl_Obj *resultPtr, *chanObjPtr;
    Tcl_Obj *resultPtr, *returnOptsPtr, *chanObjPtr;

    if ((objc != 2) && (objc != 3)) {
	Interp *iPtr;

    argerror:
	iPtr = (Interp *) interp;
	Tcl_WrongNumArgs(interp, 1, objv, "channelId ?numChars?");
466
467
468
469
470
471
472



473
474

475
476
477
478
479
480
481
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485







+
+
+


+







	 */

	if (!TclChanCaughtErrorBypass(interp, chan)) {
	    Tcl_SetObjResult(interp, Tcl_ObjPrintf(
		    "error reading \"%s\": %s",
		    TclGetString(chanObjPtr), Tcl_PosixError(interp)));
	}
	returnOptsPtr = Tcl_NewDictObj();
	Tcl_DictObjPut(NULL, returnOptsPtr, Tcl_NewStringObj("-result", -1)
	    , resultPtr);
	TclChannelRelease(chan);
	Tcl_DecrRefCount(resultPtr);
	Tcl_SetReturnOptions(interp, returnOptsPtr);
	return TCL_ERROR;
    }

    /*
     * If requested, remove the last newline in the channel if at EOF.
     */

Changes to tests/io.test.
9052
9053
9054
9055
9056
9057
9058

9059

9060
9061
9062
9063

9064

9065
9066
9067
9068
9069
9070
9071
9072
9073
9074
9075
9076
9077

9078

9079
9080
9081
9082


9083
9084
9085
9086
9087
9088
9089
9090

9091



9092
9093
9094
9095


9096
9097
9098
9099
9100
9101
9102
9103

9104
9105
9106
9107
9108
9109



























9110
9111
9112
9113
9114
9115
9116
9052
9053
9054
9055
9056
9057
9058
9059

9060
9061



9062
9063
9064
9065
9066
9067
9068
9069
9070
9071
9072
9073
9074
9075
9076
9077
9078

9079
9080
9081


9082
9083
9084
9085
9086
9087
9088
9089
9090
9091
9092

9093
9094
9095
9096
9097
9098

9099
9100
9101
9102
9103
9104
9105
9106
9107
9108
9109
9110
9111
9112
9113
9114
9115
9116
9117
9118
9119
9120
9121
9122
9123
9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141
9142
9143
9144
9145
9146
9147
9148
9149







+
-
+

-
-
-
+

+













+
-
+


-
-
+
+








+
-
+
+
+



-
+
+








+






+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
    set status [catch {read $f} cres copts]
    set d [read $f]
    set d [dict get $copts -result]
    binary scan $d H* hd
    lappend hd [catch {read $f} msg]
    close $f
    lappend hd $msg
    lappend hd $status $cres
} -cleanup {
    close $f
    removeFile io-75.6
} -match glob -result {41 1 {error reading "*": illegal byte sequence}}

test io-75.7 {invalid utf-8 encoding eof handling (-strictencoding 1)} -setup {
    set fn [makeFile {} io-75.7]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \xA1 is invalid in utf-8. -eofchar is not detected, because it comes later.
    puts -nonewline $f A\xA1\x1A
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -strictencoding 1
} -body {
    set status [catch {read $f} cres copts]
    set d [read $f]
    set d [dict get $copts -result]
    binary scan $d H* hd
    lappend hd [eof $f]
    lappend hd [catch {read $f} msg]
    lappend hd $msg
    lappend hd $status
    lappend hd $cres
    fconfigure $f -encoding iso8859-1
    lappend hd [read $f];# We changed encoding, so now we can read the \xA1
    close $f
    set hd
} -cleanup {
    removeFile io-75.7
} -match glob -result {41 0 1 {error reading "*": illegal byte sequence} ยก}

test io-75.8.incomplete {
test io-75.8 {invalid utf-8 encoding eof handling (-strictencoding 1)} -setup {
    incomplete uft-8 char after eof char is not an error (-strictencoding 1)
} -setup {
    set hd {}
    set fn [makeFile {} io-75.8]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8, but since \x1A comes first, -eofchar takes precedence.
    # \x81 is invalid and also incomplete utf-8 data, but because the eof
    # character \x1A appears first, it's not an error.
    puts -nonewline $f A\x1A\x81
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -strictencoding 1
} -body {
    set d [read $f]
    binary scan $d H* hd
    lappend hd [eof $f]
    # there should be no error on additional reads
    lappend hd [read $f]
    close $f
    set hd
} -cleanup {
    removeFile io-75.8
} -result {41 1 {}}


test io-75.8.invalid {invalid utf-8 after eof char is not an error  (-strictencoding 1)} -setup {
    set res {}
    set fn [makeFile {} io-75.8]
    set f [open $fn w+]
    fconfigure $f -encoding binary
    # \xc0\x80 is invalid utf-8 data, but because the eof character \x1A
    # appears first, it's not an error.
    puts -nonewline $f A\x1a\xc0\x80
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -strictencoding 1
} -body {
    set d [read $f]
    foreach char [split $d {}] {
	lappend res [format %x [scan $char %c]]
    }
    lappend res [eof $f]
    # there should be no error on additional reads
    lappend res [read $f]
    close $f
    set res
} -cleanup {
    removeFile io-75.8
} -result {41 1 {}}


test io-75.9 {unrepresentable character write passes and is replaced by ?} -setup {
    set fn [makeFile {} io-75.9]
    set f [open $fn w+]
    fconfigure $f -encoding iso8859-1 -strictencoding 1
} -body {
    catch {puts -nonewline $f "A\u2022"} msg
9153
9154
9155
9156
9157
9158
9159

9160

9161
9162
9163


9164
9165
9166
9167
9168
9169
9170
9186
9187
9188
9189
9190
9191
9192
9193

9194
9195


9196
9197
9198
9199
9200
9201
9202
9203
9204







+
-
+

-
-
+
+







    # In shiftjis, \x81 starts a two-byte sequence.
    # But 2nd byte \xFF is not allowed
    puts -nonewline $f A\x81\xFFA
    flush $f
    seek $f 0
    fconfigure $f -encoding shiftjis -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
    set status [catch {read $f} cres copts]
    set d [read $f]
    set d [dict get $copts -result]
    binary scan $d H* hd
    lappend hd [catch {set d [read $f]} msg]
    lappend hd $msg
    lappend hd $status
    lappend hd $cres
} -cleanup {
    close $f
    removeFile io-75.11
} -match glob -result {41 1 {error reading "*": illegal byte sequence}}

test io-75.12 {invalid utf-8 encoding read is ignored} -setup {
    set fn [makeFile {} io-75.12]
9188
9189
9190
9191
9192
9193
9194

9195

9196
9197

9198
9199

9200
9201
9202
9203
9204
9205
9206
9222
9223
9224
9225
9226
9227
9228
9229

9230
9231

9232
9233

9234
9235
9236
9237
9238
9239
9240
9241







+
-
+

-
+

-
+







    fconfigure $f -encoding binary
    # \x81 is invalid in utf-8
    puts -nonewline $f "A\x81"
    flush $f
    seek $f 0
    fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -strictencoding 1
} -body {
    set status [catch {read $f} cres copts]
    set d [read $f]
    set d [dict get $copts -result]
    binary scan $d H* hd
    lappend hd [catch {read $f} msg]
    lappend hd $status
    close $f
    lappend hd $msg
    lappend hd $cres
} -cleanup {
    removeFile io-75.13
} -match glob -result {41 1 {error reading "*": illegal byte sequence}}

# ### ### ### ######### ######### #########