Index: doc/Encoding.3 ================================================================== --- doc/Encoding.3 +++ doc/Encoding.3 @@ -103,13 +103,13 @@ routine to perform any finalization that needs to occur after the last byte is converted and then to reset to an initial state. The \fBTCL_PROFILE_*\fR bits defined in the \fBPROFILES\fR section below control the encoding profile to be used for dealing with invalid data or other errors in the encoding transform. -\fBTCL_ENCODING_STOPONERROR\fR is present for backward compatibility with -Tcl 8.6 and forces the encoding profile to \fBstrict\fR. - +The flag \fBTCL_ENCODING_STOPONERROR\fR has no effect, +it only has meaning in Tcl 8.x. +.PP Some flags bits may not be usable with some functions as noted in the function descriptions below. .AP Tcl_EncodingState *statePtr in/out Used when converting a (generally long or indefinite length) byte stream in a piece-by-piece fashion. The conversion routine stores its current @@ -588,13 +588,13 @@ to be used by OR-ing the \fBflags\fR parameter passed to the function with at most one of \fBTCL_ENCODING_PROFILE_TCL8\fR, \fBTCL_ENCODING_PROFILE_STRICT\fR or \fBTCL_ENCODING_PROFILE_REPLACE\fR. These correspond to the \fBtcl8\fR, \fBstrict\fR and \fBreplace\fR profiles respectively. If none are specified, a version-dependent default profile is used. -For Tcl 9.0, the default profile is \fBtcl8\fR. +For Tcl 9.0, the default profile is \fBstrict\fR. .PP For details about profiles, see the \fBPROFILES\fR section in the documentation of the \fBencoding\fR command. .SH "SEE ALSO" encoding(n) .SH KEYWORDS utf, encoding, convert Index: doc/chan.n ================================================================== --- doc/chan.n +++ doc/chan.n @@ -198,20 +198,19 @@ platforms it is \fBcrlf\fR for both input and output. .TP \fBbinary\fR . Like \fBlf\fR, no end-of-line translation is performed, but in addition, sets -\fB\-eofchar\fR to the empty string to disable it, sets \fB\-encoding\fR to -\fBiso8859-1\fR, and sets \fB-profile\fR to \fBstrict\fR so the the channel is -fully configured for binary input and output: Each byte read from the channel +\fB\-eofchar\fR to the empty string to disable it, and sets \fB\-encoding\fR +to \fBiso8859-1\fR. With this one setting, a channel is fully configured +for binary input and output: Each byte read from the channel becomes the Unicode character having the same value as that byte, and each character written to the channel becomes a single byte in the output. This makes it possible to work seamlessly with binary data as long as each character in the data remains in the range of 0 to 255 so that there is no distinction between binary data and text. For example, A JPEG image can be read from a such a channel, manipulated, and then written back to such a channel. - .TP \fBcr\fR . The end of a line is represented in the external data by a single carriage return character. For input, each carriage return is translated to a line Index: doc/encoding.n ================================================================== --- doc/encoding.n +++ doc/encoding.n @@ -109,13 +109,18 @@ encoding. .PP The following profiles are currently implemented. .VS "TCL8.7 TIP656" .TP +\fBstrict\fR +. +The default profile. The operation fails when invalid data for the encoding +are encountered. +.TP \fBtcl8\fR . -The default profile. Provides for behaviour identical to that of Tcl 8.6: When +Provides for behaviour identical to that of Tcl 8.6: When decoding, for encodings \fBother than utf-8\fR, each invalid byte is interpreted as the Unicode value given by that one byte. For example, the byte 0x80, which is invalid in the ASCII encoding would be mapped to the Unicode value U+0080. For \fButf-8\fR, each invalid byte that is a valid CP1252 character is interpreted as the Unicode value for that character, while each byte that is @@ -125,14 +130,10 @@ an additional special case, the sequence 0xC0 0x80 is mapped to U+0000. When encoding, each character that cannot be represented in the encoding is replaced by an encoding-dependent character, usually the question mark \fB?\fR. .TP -\fBstrict\fR -. -The operation fails when invalid data for the encoding are encountered. -.TP \fBreplace\fR . When decoding, invalid bytes are replaced by U+FFFD, the Unicode REPLACEMENT CHARACTER. @@ -178,11 +179,11 @@ .CE .PP Example 3: Get partial data and the error location: .PP .CS -% codepoints [encoding convertfrom -profile strict -failindex idx ascii AB\ex80] +% codepoints [encoding convertfrom -failindex idx ascii AB\ex80] U+000041 U+000042 % set idx 2 .CE .PP @@ -191,11 +192,11 @@ .CS % encoding convertto iso8859-1 A\eu0141 A? % encoding convertto -profile strict iso8859-1 A\eu0141 unexpected character at index 1: 'U+000141' -% encoding convertto -profile strict -failindex idx iso8859-1 A\eu0141 +% encoding convertto -failindex idx iso8859-1 A\eu0141 A % set idx 1 .CE .VE "TCL8.7 TIP607, TIP656" Index: generic/tcl.h ================================================================== --- generic/tcl.h +++ generic/tcl.h @@ -2013,18 +2013,13 @@ /* * Reserve top byte for profile values (disjoint, not a mask). In case of * changes, ensure ENCODING_PROFILE_* macros in tclInt.h are modified if * necessary. */ +#define TCL_ENCODING_PROFILE_STRICT TCL_ENCODING_STOPONERROR #define TCL_ENCODING_PROFILE_TCL8 0x01000000 -#define TCL_ENCODING_PROFILE_STRICT 0x02000000 -#define TCL_ENCODING_PROFILE_REPLACE 0x03000000 -#if TCL_MAJOR_VERSION < 9 -#define TCL_ENCODING_PROFILE_DEFAULT TCL_ENCODING_PROFILE_TCL8 -#else -#define TCL_ENCODING_PROFILE_DEFAULT TCL_ENCODING_PROFILE_TCL8 -#endif +#define TCL_ENCODING_PROFILE_REPLACE 0x02000000 /* * The following definitions are the error codes returned by the conversion * routines: * Index: generic/tclCmdAH.c ================================================================== --- generic/tclCmdAH.c +++ generic/tclCmdAH.c @@ -432,11 +432,11 @@ static const char *const options[] = {"-profile", "-failindex", NULL}; enum convertfromOptions { PROFILE, FAILINDEX } optIndex; Tcl_Encoding encoding; Tcl_Obj *dataObj; Tcl_Obj *failVarObj; - int profile = TCL_ENCODING_PROFILE_TCL8; + int profile = TCL_ENCODING_PROFILE_STRICT; /* * Possible combinations: * 1) data -> objc = 2 * 2) ?options? encoding data -> objc >= 3 Index: generic/tclEncoding.c ================================================================== --- generic/tclEncoding.c +++ generic/tclEncoding.c @@ -198,21 +198,17 @@ {"replace", TCL_ENCODING_PROFILE_REPLACE}, {"strict", TCL_ENCODING_PROFILE_STRICT}, {"tcl8", TCL_ENCODING_PROFILE_TCL8}, }; #define PROFILE_TCL8(flags_) \ - ((ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_TCL8) \ - || (ENCODING_PROFILE_GET(flags_) == 0 \ - && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_TCL8)) + (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_TCL8) + #define PROFILE_STRICT(flags_) \ - ((ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_STRICT) \ - || (ENCODING_PROFILE_GET(flags_) == 0 \ - && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_STRICT)) + (!PROFILE_TCL8(flags_) && !PROFILE_REPLACE(flags_)) + #define PROFILE_REPLACE(flags_) \ - ((ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE) \ - || (ENCODING_PROFILE_GET(flags_) == 0 \ - && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_REPLACE)) + (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE) #define UNICODE_REPLACE_CHAR ((Tcl_UniChar)0xFFFD) #define SURROGATE(c_) (((c_) & ~0x7FF) == 0xD800) #define HIGH_SURROGATE(c_) (((c_) & ~0x3FF) == 0xD800) #define LOW_SURROGATE(c_) (((c_) & ~0x3FF) == 0xDC00) @@ -1169,14 +1165,10 @@ * "flags" controls the behavior if any of the bytes in * the source buffer are invalid or cannot be represented in utf-8. * Possible flags values: * target encoding. It should be composed by OR-ing the following: * - *At most one* of TCL_ENCODING_PROFILE{DEFAULT,TCL8,STRICT} - * - TCL_ENCODING_STOPONERROR: Backward compatibility. Sets the profile - * to TCL_ENCODING_PROFILE_STRICT overriding any specified profile flags - * Any other flag bits will cause an error to be returned (for future - * compatibility) * * Results: * The return value is one of * TCL_OK: success. Converted string in *dstPtr * TCL_ERROR: error in passed parameters. Error message in interp @@ -1499,12 +1491,10 @@ * Convert a source buffer from UTF-8 to the specified encoding. * The parameter flags controls the behavior, if any of the bytes in * the source buffer are invalid or cannot be represented in the * target encoding. It should be composed by OR-ing the following: * - *At most one* of TCL_ENCODING_PROFILE{DEFAULT,TCL8,STRICT} - * - TCL_ENCODING_STOPONERROR: Backward compatibility. Sets the profile - * to TCL_ENCODING_PROFILE_STRICT overriding any specified profile flags * * Results: * The return value is one of * TCL_OK: success. Converted string in *dstPtr * TCL_ERROR: error in passed parameters. Error message in interp @@ -2450,11 +2440,10 @@ result = TCL_OK; dstLen -= TCL_UTF_MAX - 1; if (dstLen < 0) { dstLen = 0; } - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_CHAR_LIMIT) && srcLen > *dstCharsPtr) { srcLen = *dstCharsPtr; } if (srcLen > dstLen) { srcLen = dstLen; @@ -2518,11 +2507,10 @@ result = TCL_OK; srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= 6; } if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; @@ -2747,11 +2735,10 @@ const char *srcStart, *srcEnd; const char *dstEnd, *dstStart; int result, numChars, charLimit = INT_MAX; int ch = 0, bytesLeft = srcLen % 4; - flags = TclEncodingSetProfileFlags(flags); flags |= PTR2INT(clientData); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } result = TCL_OK; @@ -2927,11 +2914,10 @@ int ch, len; srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } dstStart = dst; @@ -3025,11 +3011,10 @@ const char *srcStart, *srcEnd; const char *dstEnd, *dstStart; int result, numChars, charLimit = INT_MAX; unsigned short ch = 0; - flags = TclEncodingSetProfileFlags(flags); flags |= PTR2INT(clientData); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } result = TCL_OK; @@ -3215,11 +3200,10 @@ int ch, len; srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } dstStart = dst; @@ -3321,11 +3305,10 @@ { const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd; int result, numChars, len; Tcl_UniChar ch = 0; - flags = TclEncodingSetProfileFlags(flags); flags |= PTR2INT(clientData); srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; if ((flags & TCL_ENCODING_END) == 0) { @@ -3444,11 +3427,10 @@ Tcl_UniChar ch = 0; const unsigned short *const *toUnicode; const unsigned short *pageZero; TableEncodingData *dataPtr = (TableEncodingData *)clientData; - flags = TclEncodingSetProfileFlags(flags); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } srcStart = src; srcEnd = src + srcLen; @@ -3586,11 +3568,10 @@ fromUnicode = (const unsigned short *const *) dataPtr->fromUnicode; srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } dstStart = dst; @@ -3693,11 +3674,10 @@ { const char *srcStart, *srcEnd; const char *dstEnd, *dstStart; int result, numChars, charLimit = INT_MAX; - flags = TclEncodingSetProfileFlags(flags); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } srcStart = src; srcEnd = src + srcLen; @@ -3787,11 +3767,10 @@ Tcl_UniChar ch = 0; srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } dstStart = dst; @@ -3935,11 +3914,10 @@ const unsigned short *const *tableToUnicode; const Encoding *encodingPtr; int state, result, numChars, charLimit = INT_MAX; const char *dstStart, *dstEnd; - flags = TclEncodingSetProfileFlags(flags); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } result = TCL_OK; tablePrefixBytes = NULL; @@ -4165,11 +4143,10 @@ result = TCL_OK; srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } dstStart = dst; @@ -4610,52 +4587,10 @@ Tcl_SetErrorCode( interp, "TCL", "ENCODING", "PROFILEID", NULL); } return NULL; } - -/* - *------------------------------------------------------------------------ - * - * TclEncodingSetProfileFlags -- - * - * Maps the flags supported in the encoding C API's to internal flags. - * - * For backward compatibility reasons, TCL_ENCODING_STOPONERROR is - * is mapped to the TCL_ENCODING_PROFILE_STRICT overwriting any profile - * specified. - * - * If no profile or an invalid profile is specified, it is set to - * the default. - * - * Results: - * Internal encoding flag mask. - * - * Side effects: - * None. - * - *------------------------------------------------------------------------ - */ -int TclEncodingSetProfileFlags(int flags) -{ - if (flags & TCL_ENCODING_STOPONERROR) { - ENCODING_PROFILE_SET(flags, TCL_ENCODING_PROFILE_STRICT); - } else { - int profile = ENCODING_PROFILE_GET(flags); - switch (profile) { - case TCL_ENCODING_PROFILE_TCL8: - case TCL_ENCODING_PROFILE_STRICT: - case TCL_ENCODING_PROFILE_REPLACE: - break; - case 0: /* Unspecified by caller */ - default: - ENCODING_PROFILE_SET(flags, TCL_ENCODING_PROFILE_DEFAULT); - break; - } - } - return flags; -} /* *------------------------------------------------------------------------ * * TclGetEncodingProfiles -- Index: generic/tclIO.c ================================================================== --- generic/tclIO.c +++ generic/tclIO.c @@ -1678,16 +1678,12 @@ name = Tcl_GetEncodingName(NULL); statePtr->encoding = Tcl_GetEncoding(NULL, name); statePtr->inputEncodingState = NULL; statePtr->inputEncodingFlags = TCL_ENCODING_START; - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags, - TCL_ENCODING_PROFILE_DEFAULT); statePtr->outputEncodingState = NULL; statePtr->outputEncodingFlags = TCL_ENCODING_START; - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags, - TCL_ENCODING_PROFILE_DEFAULT); /* * Set the channel up initially in AUTO input translation mode to accept * "\n", "\r" and "\r\n". Output translation mode is set to a platform * specific default value. The eofChar is set to 0 for both input and @@ -8165,16 +8161,10 @@ Tcl_Encoding encoding; int profile; if ((newValue[0] == '\0') || (strcmp(newValue, "binary") == 0)) { encoding = Tcl_GetEncoding(NULL, "iso8859-1"); - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->inputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->outputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); } else { encoding = Tcl_GetEncoding(interp, newValue); if (encoding == NULL) { return TCL_ERROR; } @@ -8279,16 +8269,10 @@ } else if (strcmp(readMode, "binary") == 0) { translation = TCL_TRANSLATE_LF; statePtr->inEofChar = 0; Tcl_FreeEncoding(statePtr->encoding); statePtr->encoding = Tcl_GetEncoding(NULL, "iso8859-1"); - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->inputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->outputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); } else if (strcmp(readMode, "lf") == 0) { translation = TCL_TRANSLATE_LF; } else if (strcmp(readMode, "cr") == 0) { translation = TCL_TRANSLATE_CR; } else if (strcmp(readMode, "crlf") == 0) { @@ -8334,16 +8318,10 @@ } } else if (strcmp(writeMode, "binary") == 0) { statePtr->outputTranslation = TCL_TRANSLATE_LF; Tcl_FreeEncoding(statePtr->encoding); statePtr->encoding = Tcl_GetEncoding(NULL, "iso8859-1"); - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->inputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->outputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); } else if (strcmp(writeMode, "lf") == 0) { statePtr->outputTranslation = TCL_TRANSLATE_LF; } else if (strcmp(writeMode, "cr") == 0) { statePtr->outputTranslation = TCL_TRANSLATE_CR; } else if (strcmp(writeMode, "crlf") == 0) { Index: generic/tclIOSock.c ================================================================== --- generic/tclIOSock.c +++ generic/tclIOSock.c @@ -73,11 +73,15 @@ if (Tcl_GetInt(NULL, string, portPtr) != TCL_OK) { /* * Don't bother translating 'proto' to native. */ - native = Tcl_UtfToExternalDString(NULL, string, -1, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, string, -1, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return TCL_ERROR; + } + native = Tcl_DStringValue(&ds); sp = getservbyname(native, proto); /* INTL: Native. */ Tcl_DStringFree(&ds); if (sp != NULL) { *portPtr = ntohs((unsigned short) sp->s_port); return TCL_OK; @@ -182,11 +186,15 @@ const char *family = NULL; Tcl_DString ds; int result; if (host != NULL) { - native = Tcl_UtfToExternalDString(NULL, host, -1, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, host, -1, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return 0; + } + native = Tcl_DStringValue(&ds); } /* * Workaround for OSX's apparent inability to resolve "localhost", "0" * when the loopback device is the only available network interface. Index: generic/tclInt.h ================================================================== --- generic/tclInt.h +++ generic/tclInt.h @@ -3033,11 +3033,10 @@ TclEncodingProfileNameToId(Tcl_Interp *interp, const char *profileName, int *profilePtr); MODULE_SCOPE const char *TclEncodingProfileIdToName(Tcl_Interp *interp, int profileId); -MODULE_SCOPE int TclEncodingSetProfileFlags(int flags); MODULE_SCOPE void TclGetEncodingProfiles(Tcl_Interp *interp); /* * TIP #233 (Virtualized Time) * Data for the time hooks, if any. Index: generic/tclZipfs.c ================================================================== --- generic/tclZipfs.c +++ generic/tclZipfs.c @@ -2539,11 +2539,15 @@ /* * Convert to encoded form. Note that we use strlen() here; if someone's * crazy enough to embed NULs in filenames, they deserve what they get! */ - zpathExt = Tcl_UtfToExternalDString(ZipFS.utf8, zpathTcl, -1, &zpathDs); + if (Tcl_UtfToExternalDStringEx(interp, ZipFS.utf8, zpathTcl, TCL_INDEX_NONE, 0, &zpathDs, NULL) != TCL_OK) { + Tcl_DStringFree(&zpathDs); + return TCL_ERROR; + } + zpathExt = Tcl_DStringValue(&zpathDs); zpathlen = strlen(zpathExt); if (zpathlen + ZIP_CENTRAL_HEADER_LEN > bufsize) { Tcl_SetObjResult(interp, Tcl_ObjPrintf( "path too long for \"%s\"", TclGetString(pathObj))); ZIPFS_ERROR_CODE(interp, "PATH_LEN"); @@ -3208,11 +3212,15 @@ if (!hPtr) { continue; } z = (ZipEntry *) Tcl_GetHashValue(hPtr); - name = Tcl_UtfToExternalDString(ZipFS.utf8, z->name, TCL_INDEX_NONE, &ds); + if (Tcl_UtfToExternalDStringEx(interp, ZipFS.utf8, z->name, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) { + ret = TCL_ERROR; + goto done; + } + name = Tcl_DStringValue(&ds); len = Tcl_DStringLength(&ds); SerializeCentralDirectoryEntry(start, end, (unsigned char *) buf, z, len); if ((Tcl_Write(out, buf, ZIP_CENTRAL_HEADER_LEN) != ZIP_CENTRAL_HEADER_LEN) Index: library/http/http.tcl ================================================================== --- library/http/http.tcl +++ library/http/http.tcl @@ -1745,11 +1745,11 @@ Log socket delay $delay - token $token } fconfigure $sock -translation {auto crlf} \ -buffersize $state(-blocksize) if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 + fconfigure $sock -profile replace } ##Log socket opened, DONE fconfigure - token $token } Log "Using $sock for $state(socketinfo) - token $token" \ @@ -2166,11 +2166,11 @@ # We are concerned here with the request (write) not the response (read). lassign [fconfigure $sock -translation] trRead trWrite fconfigure $sock -translation [list $trRead crlf] \ -buffersize $state(-blocksize) if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 + fconfigure $sock -profile replace } # The following is disallowed in safe interpreters, but the socket is # already in non-blocking mode in that case. @@ -2559,11 +2559,11 @@ #Log ---- $state(socketinfo) >> conn to $token for HTTP response lassign [fconfigure $sock -translation] trRead trWrite fconfigure $sock -translation [list auto $trWrite] \ -buffersize $state(-blocksize) if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 + fconfigure $sock -profile replace } Log ^D$tk begin receiving response - token $token coroutine ${token}--EventCoroutine http::Event $sock $token if {[info exists state(-handler)] || [info exists state(-progress)]} { @@ -4553,11 +4553,11 @@ # encodings for. set enc [CharsetToEncoding $state(charset)] if {$enc ne "binary"} { if {[package vsatisfies [package provide Tcl] 9.0-]} { - set state(body) [encoding convertfrom -profile tcl8 $enc $state(body)] + set state(body) [encoding convertfrom -profile replace $enc $state(body)] } else { set state(body) [encoding convertfrom $enc $state(body)] } } @@ -4640,11 +4640,11 @@ set enc [CharsetToEncoding $res] if {$enc eq "binary"} { return 0 } if {[package vsatisfies [package provide Tcl] 9.0-]} { - set state(body) [encoding convertfrom -profile tcl8 $enc $state(body)] + set state(body) [encoding convertfrom -profile replace $enc $state(body)] } else { set state(body) [encoding convertfrom $enc $state(body)] } set state(body) [string map {\r\n \n \r \n} $state(body)] set state(type) application/xml @@ -4725,11 +4725,11 @@ # The spec says: "non-alphanumeric characters are replaced by '%HH'". Use # a pre-computed map and [string map] to do the conversion (much faster # than [regsub]/[subst]). [Bug 1020491] if {[package vsatisfies [package provide Tcl] 9.0-]} { - set string [encoding convertto -profile tcl8 $http(-urlencoding) $string] + set string [encoding convertto -profile replace $http(-urlencoding) $string] } else { set string [encoding convertto $http(-urlencoding) $string] } return [string map $formMap $string] } Index: tests/encoding.test ================================================================== --- tests/encoding.test +++ tests/encoding.test @@ -575,11 +575,11 @@ } -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\x00'} test encoding-16.23 {Utf16ToUtfProc, strict, bug [db7a085bd9]} -body { encoding convertfrom -profile strict utf-16le \x00\xDC } -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\x00'} test encoding-16.24 {Utf32ToUtfProc} -body { - encoding convertfrom utf-32 "\xFF\xFF\xFF\xFF" + encoding convertfrom -profile tcl8 utf-32 "\xFF\xFF\xFF\xFF" } -result \uFFFD test {encoding-16.25 strict} {Utf32ToUtfProc} -body { encoding convertfrom -profile strict utf-32 "\x01\x00\x00\x01" } -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\x01'} test {encoding-16.25 tcl8} {Utf32ToUtfProc} -body { Index: tests/encodingVectors.tcl ================================================================== --- tests/encodingVectors.tcl +++ tests/encodingVectors.tcl @@ -8,11 +8,11 @@ # vectors. # # List of defined encoding profiles set encProfiles {tcl8 strict replace} -set encDefaultProfile tcl8; # Should reflect the default from implementation +set encDefaultProfile strict; # Should reflect the default from implementation # encValidStrings - Table of valid strings. # # Each row is # The pair should be unique for generated test ids to be unique. Index: tests/ioCmd.test ================================================================== --- tests/ioCmd.test +++ tests/ioCmd.test @@ -238,15 +238,15 @@ } -result {bad value for -translation: must be one of auto, binary, cr, lf, crlf, or platform} test iocmd-8.7 {fconfigure command} -setup { file delete $path(test1) } -body { set f1 [open $path(test1) w] - fconfigure $f1 -translation lf -eofchar {} -encoding utf-16 -profile tcl8 + fconfigure $f1 -translation lf -eofchar {} -encoding utf-16 fconfigure $f1 } -cleanup { catch {close $f1} -} -result {-blocking 1 -buffering full -buffersize 4096 -encoding utf-16 -eofchar {} -profile tcl8 -translation lf} +} -result {-blocking 1 -buffering full -buffersize 4096 -encoding utf-16 -eofchar {} -profile strict -translation lf} test iocmd-8.8 {fconfigure command} -setup { file delete $path(test1) set x {} } -body { set f1 [open $path(test1) w] @@ -260,15 +260,15 @@ test iocmd-8.9 {fconfigure command} -setup { file delete $path(test1) } -body { set f1 [open $path(test1) w] fconfigure $f1 -translation binary -buffering none -buffersize 4040 \ - -eofchar {} -encoding binary -profile tcl8 + -eofchar {} -encoding binary fconfigure $f1 } -cleanup { catch {close $f1} -} -result {-blocking 1 -buffering none -buffersize 4040 -encoding iso8859-1 -eofchar {} -profile tcl8 -translation lf} +} -result {-blocking 1 -buffering none -buffersize 4040 -encoding iso8859-1 -eofchar {} -profile strict -translation lf} test iocmd-8.10 {fconfigure command} -returnCodes error -body { fconfigure a b } -result {can not find channel named "a"} set path(fconfigure.dummy) [makeFile {} fconfigure.dummy] test iocmd-8.11 {fconfigure command} -body { Index: tests/utfext.test ================================================================== --- tests/utfext.test +++ tests/utfext.test @@ -74,13 +74,13 @@ # % testencoding Tcl_ExternalToUtf utf-8 abcdefgh {start end noterminate charlimit} {} 20 rv wv cv # nospace {} abcÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ test TableToUtf-bug-5be203d6ca {Bug 5be203d6ca - truncated prefix in table encoding} -body { set src \x82\x4F\x82\x50\x82 - lassign [testencoding Tcl_ExternalToUtf shiftjis $src {start} 0 16 srcRead dstWritten charsWritten] buf - set result [list [testencoding Tcl_ExternalToUtf shiftjis $src {start} 0 16 srcRead dstWritten charsWritten] $srcRead $dstWritten $charsWritten] - lappend result {*}[list [testencoding Tcl_ExternalToUtf shiftjis [string range $src $srcRead end] {end} 0 10 srcRead dstWritten charsWritten] $srcRead $dstWritten $charsWritten] + lassign [testencoding Tcl_ExternalToUtf shiftjis $src {start profiletcl8} 0 16 srcRead dstWritten charsWritten] buf + set result [list [testencoding Tcl_ExternalToUtf shiftjis $src {start profiletcl8} 0 16 srcRead dstWritten charsWritten] $srcRead $dstWritten $charsWritten] + lappend result {*}[list [testencoding Tcl_ExternalToUtf shiftjis [string range $src $srcRead end] {end profiletcl8} 0 10 srcRead dstWritten charsWritten] $srcRead $dstWritten $charsWritten] } -result [list [list multibyte 0 \xEF\xBC\x90\xEF\xBC\x91\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF] 4 6 2 [list ok 0 \xC2\x82\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF] 1 2 1] ::tcltest::cleanupTests return Index: unix/tclLoadDl.c ================================================================== --- unix/tclLoadDl.c +++ unix/tclLoadDl.c @@ -106,11 +106,15 @@ */ Tcl_DString ds; const char *fileName = TclGetString(pathPtr); - native = Tcl_UtfToExternalDString(NULL, fileName, TCL_INDEX_NONE, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, fileName, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return TCL_ERROR; + } + native = Tcl_DStringValue(&ds); /* * Use (RTLD_NOW|RTLD_LOCAL) as default, see [Bug #3216070] */ handle = dlopen(native, dlopenflags); Tcl_DStringFree(&ds); @@ -177,11 +181,15 @@ * Some platforms still add an underscore to the beginning of symbol * names. If we can't find a name without an underscore, try again with * the underscore. */ - native = Tcl_UtfToExternalDString(NULL, symbol, TCL_INDEX_NONE, &ds); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, symbol, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return NULL; + } + native = Tcl_DStringValue(&ds); proc = dlsym(handle, native); /* INTL: Native. */ if (proc == NULL) { Tcl_DStringInit(&newName); TclDStringAppendLiteral(&newName, "_"); native = Tcl_DStringAppend(&newName, native, TCL_INDEX_NONE); Index: unix/tclUnixFCmd.c ================================================================== --- unix/tclUnixFCmd.c +++ unix/tclUnixFCmd.c @@ -760,32 +760,39 @@ Tcl_DString srcString, dstString; int ret; Tcl_Obj *transPtr; transPtr = Tcl_FSGetTranslatedPath(NULL,srcPathPtr); - Tcl_UtfToExternalDStringEx(NULL, NULL, - (transPtr != NULL ? TclGetString(transPtr) : NULL), - -1, TCL_ENCODING_PROFILE_TCL8, &srcString, NULL); - if (transPtr != NULL) { - Tcl_DecrRefCount(transPtr); - } - transPtr = Tcl_FSGetTranslatedPath(NULL,destPathPtr); - Tcl_UtfToExternalDStringEx(NULL, NULL, - (transPtr != NULL ? TclGetString(transPtr) : NULL), - -1, TCL_ENCODING_PROFILE_TCL8, &dstString, NULL); - if (transPtr != NULL) { - Tcl_DecrRefCount(transPtr); - } - - ret = TraverseUnixTree(TraversalCopy, &srcString, &dstString, &ds, 0); - - Tcl_DStringFree(&srcString); - Tcl_DStringFree(&dstString); - - if (ret != TCL_OK) { - *errorPtr = Tcl_NewStringObj(Tcl_DStringValue(&ds), TCL_INDEX_NONE); - Tcl_DStringFree(&ds); + ret = Tcl_UtfToExternalDStringEx(NULL, NULL, + (transPtr != NULL ? TclGetString(transPtr) : NULL), + -1, 0, &srcString, NULL); + if (transPtr != NULL) { + Tcl_DecrRefCount(transPtr); + } + if (ret != TCL_OK) { + *errorPtr = srcPathPtr; + } else { + transPtr = Tcl_FSGetTranslatedPath(NULL,destPathPtr); + ret = Tcl_UtfToExternalDStringEx(NULL, NULL, + (transPtr != NULL ? TclGetString(transPtr) : NULL), + -1, TCL_ENCODING_PROFILE_TCL8, &dstString, NULL); + if (transPtr != NULL) { + Tcl_DecrRefCount(transPtr); + } + if (ret != TCL_OK) { + *errorPtr = destPathPtr; + } else { + ret = TraverseUnixTree(TraversalCopy, &srcString, &dstString, &ds, 0); + /* Note above call only sets ds on error */ + if (ret != TCL_OK) { + *errorPtr = Tcl_DStringToObj(&ds); + } + Tcl_DStringFree(&dstString); + } + Tcl_DStringFree(&srcString); + } + if (ret != TCL_OK) { Tcl_IncrRefCount(*errorPtr); } return ret; } @@ -824,22 +831,28 @@ Tcl_DString ds; Tcl_DString pathString; int ret; Tcl_Obj *transPtr = Tcl_FSGetTranslatedPath(NULL, pathPtr); - Tcl_UtfToExternalDStringEx(NULL, NULL, + ret = Tcl_UtfToExternalDStringEx(NULL, NULL, (transPtr != NULL ? TclGetString(transPtr) : NULL), -1, TCL_ENCODING_PROFILE_TCL8, &pathString, NULL); if (transPtr != NULL) { Tcl_DecrRefCount(transPtr); } - ret = DoRemoveDirectory(&pathString, recursive, &ds); - Tcl_DStringFree(&pathString); - if (ret != TCL_OK) { - *errorPtr = Tcl_NewStringObj(Tcl_DStringValue(&ds), TCL_INDEX_NONE); - Tcl_DStringFree(&ds); + *errorPtr = pathPtr; + } else { + ret = DoRemoveDirectory(&pathString, recursive, &ds); + Tcl_DStringFree(&pathString); + /* Note above call only sets ds on error */ + if (ret != TCL_OK) { + *errorPtr = Tcl_DStringToObj(&ds); + } + } + + if (ret != TCL_OK) { Tcl_IncrRefCount(*errorPtr); } return ret; } @@ -884,11 +897,11 @@ } result = TCL_OK; if ((errno != EEXIST) || (recursive == 0)) { if (errorPtr != NULL) { - Tcl_ExternalToUtfDStringEx(NULL, NULL, path, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, errorPtr, NULL); + Tcl_ExternalToUtfDStringEx(NULL, NULL, path, TCL_INDEX_NONE, 0, errorPtr, NULL); } result = TCL_ERROR; } /* @@ -1133,11 +1146,11 @@ #endif /* !HAVE_FTS */ end: if (errfile != NULL) { if (errorPtr != NULL) { - Tcl_ExternalToUtfDStringEx(NULL, NULL, errfile, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, errorPtr, NULL); + Tcl_ExternalToUtfDStringEx(NULL, NULL, errfile, TCL_INDEX_NONE, 0, errorPtr, NULL); } result = TCL_ERROR; } #ifdef HAVE_FTS if (fts != NULL) { @@ -1204,11 +1217,11 @@ * get here. */ if (errorPtr != NULL) { Tcl_ExternalToUtfDStringEx(NULL, NULL, Tcl_DStringValue(dstPtr), - Tcl_DStringLength(dstPtr), TCL_ENCODING_PROFILE_TCL8, errorPtr, NULL); + Tcl_DStringLength(dstPtr), 0, errorPtr, NULL); } return TCL_ERROR; } /* @@ -1255,11 +1268,11 @@ } break; } if (errorPtr != NULL) { Tcl_ExternalToUtfDStringEx(NULL, NULL, Tcl_DStringValue(srcPtr), - Tcl_DStringLength(srcPtr), TCL_ENCODING_PROFILE_TCL8, errorPtr, NULL); + Tcl_DStringLength(srcPtr), 0, errorPtr, NULL); } return TCL_ERROR; } /* @@ -1422,11 +1435,11 @@ if (pwPtr == NULL) { TclNewIntObj(*attributePtrPtr, statBuf.st_uid); } else { Tcl_DString ds; - Tcl_ExternalToUtfDStringEx(NULL, NULL, pwPtr->pw_name, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, &ds, NULL); + Tcl_ExternalToUtfDStringEx(NULL, NULL, pwPtr->pw_name, TCL_INDEX_NONE, 0, &ds, NULL); *attributePtrPtr = Tcl_DStringToObj(&ds); } return TCL_OK; } @@ -1506,11 +1519,15 @@ const char *string; Tcl_Size length; string = Tcl_GetStringFromObj(attributePtr, &length); - native = Tcl_UtfToExternalDString(NULL, string, length, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, string, length, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return TCL_ERROR; + } + native = Tcl_DStringValue(&ds); groupPtr = TclpGetGrNam(native); /* INTL: Native. */ Tcl_DStringFree(&ds); if (groupPtr == NULL) { if (interp != NULL) { @@ -1573,11 +1590,15 @@ const char *string; Tcl_Size length; string = Tcl_GetStringFromObj(attributePtr, &length); - native = Tcl_UtfToExternalDString(NULL, string, length, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, string, length, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return TCL_ERROR; + } + native = Tcl_DStringValue(&ds); pwPtr = TclpGetPwNam(native); /* INTL: Native. */ Tcl_DStringFree(&ds); if (pwPtr == NULL) { if (interp != NULL) { @@ -1933,11 +1954,11 @@ *--------------------------------------------------------------------------- */ int TclpObjNormalizePath( - TCL_UNUSED(Tcl_Interp *), + Tcl_Interp *interp, Tcl_Obj *pathPtr, /* An unshared object containing the path to * normalize. */ int nextCheckpoint) /* offset to start at in pathPtr. Must either * be 0 or the offset of a directory separator * at the end of a path part that is already @@ -1967,12 +1988,16 @@ */ const char *lastDir = strrchr(currentPathEndPosition, '/'); if (lastDir != NULL) { - nativePath = Tcl_UtfToExternalDString(NULL, path, - lastDir-path, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, path, + lastDir-path, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return -1; + } + nativePath = Tcl_DStringValue(&ds); if (Realpath(nativePath, normPath) != NULL) { if (*nativePath != '/' && *normPath == '/') { /* * realpath transformed a relative path into an * absolute path. Fall back to the long way. @@ -2003,12 +2028,16 @@ * Reached directory separator. */ int accessOk; - nativePath = Tcl_UtfToExternalDString(NULL, path, - currentPathEndPosition - path, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, path, + currentPathEndPosition - path, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return -1; + } + nativePath = Tcl_DStringValue(&ds); accessOk = access(nativePath, F_OK); Tcl_DStringFree(&ds); if (accessOk != 0) { /* @@ -2048,11 +2077,15 @@ */ return 0; } - nativePath = Tcl_UtfToExternalDString(NULL, path,nextCheckpoint, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, path,nextCheckpoint, 0, &ds, NULL)) { + Tcl_DStringFree(&ds); + return -1; + } + nativePath = Tcl_DStringValue(&ds); if (Realpath(nativePath, normPath) != NULL) { Tcl_Size newNormLen; wholeStringOk: newNormLen = strlen(normPath); @@ -2084,11 +2117,11 @@ /* * Free the original path and replace it with the normalized path. */ Tcl_DStringFree(&ds); - Tcl_ExternalToUtfDStringEx(NULL, NULL, normPath, newNormLen, TCL_ENCODING_PROFILE_TCL8, &ds, NULL); + Tcl_ExternalToUtfDStringEx(NULL, NULL, normPath, newNormLen, 0, &ds, NULL); if (path[nextCheckpoint] != '\0') { /* * Append the remaining path components. */ @@ -2172,26 +2205,31 @@ const char *string; int fd; Tcl_Size length; /* - * We should also check against making more then TMP_MAX of these. + * We should also check against making more than TMP_MAX of these. */ if (dirObj) { string = Tcl_GetStringFromObj(dirObj, &length); - Tcl_UtfToExternalDStringEx(NULL, NULL, string, length, TCL_ENCODING_PROFILE_TCL8, &templ, NULL); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, string, length, 0, &templ, NULL) != TCL_OK) { + return -1; + } } else { Tcl_DStringInit(&templ); Tcl_DStringAppend(&templ, DefaultTempDir(), TCL_INDEX_NONE); /* INTL: native */ } TclDStringAppendLiteral(&templ, "/"); if (basenameObj) { string = Tcl_GetStringFromObj(basenameObj, &length); - Tcl_UtfToExternalDStringEx(NULL, NULL, string, length, TCL_ENCODING_PROFILE_TCL8, &tmp, NULL); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, string, length, 0, &tmp, NULL) != TCL_OK) { + Tcl_DStringFree(&tmp); + return -1; + } TclDStringAppendDString(&templ, &tmp); Tcl_DStringFree(&tmp); } else { TclDStringAppendLiteral(&templ, "tcl"); } @@ -2199,11 +2237,14 @@ TclDStringAppendLiteral(&templ, "_XXXXXX"); #ifdef HAVE_MKSTEMPS if (extensionObj) { string = Tcl_GetStringFromObj(extensionObj, &length); - Tcl_UtfToExternalDStringEx(NULL, NULL, string, length, TCL_ENCODING_PROFILE_TCL8, &tmp, NULL); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, string, length, 0, &tmp, NULL) != TCL_OK) { + Tcl_DStringFree(&templ); + return -1; + } TclDStringAppendDString(&templ, &tmp); fd = mkstemps(Tcl_DStringValue(&templ), Tcl_DStringLength(&tmp)); Tcl_DStringFree(&tmp); } else #endif @@ -2215,12 +2256,15 @@ Tcl_DStringFree(&templ); return -1; } if (resultingNameObj) { - Tcl_ExternalToUtfDStringEx(NULL, NULL, Tcl_DStringValue(&templ), - Tcl_DStringLength(&templ), TCL_ENCODING_PROFILE_TCL8, &tmp, NULL); + if (Tcl_ExternalToUtfDStringEx(NULL, NULL, Tcl_DStringValue(&templ), + Tcl_DStringLength(&templ), 0, &tmp, NULL) != TCL_OK) { + Tcl_DStringFree(&templ); + return -1; + } Tcl_SetStringObj(resultingNameObj, Tcl_DStringValue(&tmp), Tcl_DStringLength(&tmp)); Tcl_DStringFree(&tmp); } else { /* @@ -2302,11 +2346,13 @@ * some defaults. */ if (dirObj) { string = TclGetString(dirObj); - Tcl_UtfToExternalDStringEx(NULL, NULL, string, dirObj->length, TCL_ENCODING_PROFILE_TCL8, &templ, NULL); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, string, dirObj->length, 0, &templ, NULL) != TCL_OK) { + return NULL; + } } else { Tcl_DStringInit(&templ); Tcl_DStringAppend(&templ, DefaultTempDir(), TCL_INDEX_NONE); /* INTL: native */ } @@ -2315,11 +2361,14 @@ } if (basenameObj) { string = TclGetString(basenameObj); if (basenameObj->length) { - Tcl_UtfToExternalDStringEx(NULL, NULL, string, basenameObj->length, TCL_ENCODING_PROFILE_TCL8, &tmp, NULL); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, string, basenameObj->length, 0, &tmp, NULL) != TCL_OK) { + Tcl_DStringFree(&templ); + return NULL; + } TclDStringAppendDString(&templ, &tmp); Tcl_DStringFree(&tmp); } else { TclDStringAppendLiteral(&templ, DEFAULT_TEMP_DIR_PREFIX); } @@ -2340,12 +2389,15 @@ /* * The template has been updated. Tell the caller what it was. */ - Tcl_ExternalToUtfDStringEx(NULL, NULL, Tcl_DStringValue(&templ), - Tcl_DStringLength(&templ), TCL_ENCODING_PROFILE_TCL8, &tmp, NULL); + if (Tcl_ExternalToUtfDStringEx(NULL, NULL, Tcl_DStringValue(&templ), + Tcl_DStringLength(&templ), 0, &tmp, NULL) != TCL_OK) { + Tcl_DStringFree(&templ); + return NULL; + } Tcl_DStringFree(&templ); return Tcl_DStringToObj(&tmp); } #if defined(__CYGWIN__) Index: unix/tclUnixFile.c ================================================================== --- unix/tclUnixFile.c +++ unix/tclUnixFile.c @@ -306,11 +306,17 @@ /* * Now open the directory for reading and iterate over the contents. */ - native = Tcl_UtfToExternalDString(NULL, dirName, TCL_INDEX_NONE, &ds); + if (Tcl_UtfToExternalDStringEx(interp, NULL, dirName, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&dsOrig); + Tcl_DStringFree(&ds); + Tcl_DecrRefCount(fileNamePtr); + return TCL_ERROR; + } + native = Tcl_DStringValue(&ds); if ((TclOSstat(native, &statBuf) != 0) /* INTL: Native. */ || !S_ISDIR(statBuf.st_mode)) { Tcl_DStringFree(&dsOrig); Tcl_DStringFree(&ds); @@ -370,12 +376,16 @@ /* * Now check to see if the file matches, according to both type * and pattern. If so, add the file to the result. */ - utfname = Tcl_ExternalToUtfDString(NULL, entryPtr->d_name, TCL_INDEX_NONE, - &utfDs); + if (Tcl_ExternalToUtfDStringEx(interp, NULL, entryPtr->d_name, TCL_INDEX_NONE, + 0, &utfDs, NULL) != TCL_OK) { + matchResult = -1; + break; + } + utfname = Tcl_DStringValue(&utfDs); if (Tcl_StringCaseMatch(utfname, pattern, 0)) { int typeOk = 1; if (types != NULL) { Tcl_DStringSetLength(&ds, nativeDirLen); @@ -597,19 +607,29 @@ Tcl_DString *bufferPtr) /* Uninitialized or free DString filled with * name of user's home directory. */ { struct passwd *pwPtr; Tcl_DString ds; - const char *native = Tcl_UtfToExternalDString(NULL, name, TCL_INDEX_NONE, &ds); + const char *native; + + if (Tcl_UtfToExternalDStringEx(NULL, NULL, name, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return NULL; + } + native = Tcl_DStringValue(&ds); pwPtr = TclpGetPwNam(native); /* INTL: Native. */ Tcl_DStringFree(&ds); if (pwPtr == NULL) { return NULL; } - return Tcl_ExternalToUtfDString(NULL, pwPtr->pw_dir, TCL_INDEX_NONE, bufferPtr); + if (Tcl_ExternalToUtfDStringEx(NULL, NULL, pwPtr->pw_dir, TCL_INDEX_NONE, 0, bufferPtr, NULL) != TCL_OK) { + return NULL; + } else { + return Tcl_DStringValue(bufferPtr); + } } /* *--------------------------------------------------------------------------- * @@ -783,11 +803,14 @@ "error getting working directory name: %s", Tcl_PosixError(interp))); } return NULL; } - return Tcl_ExternalToUtfDString(NULL, buffer, TCL_INDEX_NONE, bufferPtr); + if (Tcl_ExternalToUtfDStringEx(interp, NULL, buffer, TCL_INDEX_NONE, 0, bufferPtr, NULL) != TCL_OK) { + return NULL; + } + return Tcl_DStringValue(bufferPtr); } /* *--------------------------------------------------------------------------- * @@ -814,27 +837,32 @@ Tcl_DString *linkPtr) /* Uninitialized or free DString filled with * contents of link (UTF-8). */ { #ifndef DJGPP char link[MAXPATHLEN]; - ssize_t length; + Tcl_Size length; const char *native; Tcl_DString ds; - native = Tcl_UtfToExternalDString(NULL, path, TCL_INDEX_NONE, &ds); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, path, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return NULL; + } + native = Tcl_DStringValue(&ds); length = readlink(native, link, sizeof(link)); /* INTL: Native. */ Tcl_DStringFree(&ds); if (length < 0) { return NULL; } - Tcl_ExternalToUtfDStringEx(NULL, NULL, link, (size_t)length, TCL_ENCODING_PROFILE_TCL8, linkPtr, NULL); - return Tcl_DStringValue(linkPtr); -#else - return NULL; + if (Tcl_ExternalToUtfDStringEx(NULL, NULL, link, length, 0, linkPtr, NULL) == TCL_OK) { + return Tcl_DStringValue(linkPtr); + } #endif /* !DJGPP */ + + return NULL; } /* *---------------------------------------------------------------------- * @@ -960,11 +988,15 @@ transPtr = Tcl_FSGetTranslatedPath(NULL, toPtr); if (transPtr == NULL) { return NULL; } target = Tcl_GetStringFromObj(transPtr, &length); - target = Tcl_UtfToExternalDString(NULL, target, length, &ds); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, target, length, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return NULL; + } + target = Tcl_DStringValue(&ds); Tcl_DecrRefCount(transPtr); if (symlink(target, src) != 0) { toPtr = NULL; } @@ -995,11 +1027,13 @@ length = readlink((const char *)Tcl_FSGetNativePath(pathPtr), link, sizeof(link)); if (length < 0) { return NULL; } - Tcl_ExternalToUtfDStringEx(NULL, NULL, link, (size_t)length, TCL_ENCODING_PROFILE_TCL8, &ds, NULL); + if (Tcl_ExternalToUtfDStringEx(NULL, NULL, link, (size_t)length, 0, &ds, NULL) != TCL_OK) { + return NULL; + } linkPtr = Tcl_DStringToObj(&ds); Tcl_IncrRefCount(linkPtr); return linkPtr; } } @@ -1060,11 +1094,11 @@ TclpNativeToNormalized( void *clientData) { Tcl_DString ds; - Tcl_ExternalToUtfDStringEx(NULL, NULL, (const char *) clientData, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, &ds, NULL); + Tcl_ExternalToUtfDStringEx(NULL, NULL, (const char *) clientData, TCL_INDEX_NONE, 0, &ds, NULL); return Tcl_DStringToObj(&ds); } /* *--------------------------------------------------------------------------- @@ -1114,11 +1148,15 @@ } Tcl_IncrRefCount(validPathPtr); } str = Tcl_GetStringFromObj(validPathPtr, &len); - Tcl_UtfToExternalDStringEx(NULL, NULL, str, len, TCL_ENCODING_PROFILE_TCL8, &ds, NULL); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, str, len, 0, &ds, NULL) != TCL_OK) { + Tcl_DecrRefCount(validPathPtr); + Tcl_DStringFree(&ds); + return NULL; + } len = Tcl_DStringLength(&ds) + sizeof(char); if (strlen(Tcl_DStringValue(&ds)) < len - sizeof(char)) { /* See bug [3118489]: NUL in filenames */ Tcl_DecrRefCount(validPathPtr); Tcl_DStringFree(&ds); Index: unix/tclUnixPipe.c ================================================================== --- unix/tclUnixPipe.c +++ unix/tclUnixPipe.c @@ -139,11 +139,15 @@ { int fd; const char *native; Tcl_DString ds; - native = Tcl_UtfToExternalDString(NULL, fname, TCL_INDEX_NONE, &ds); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, fname, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) { + Tcl_DStringFree(&ds); + return NULL; + } + native = Tcl_DStringValue(&ds); fd = TclOSopen(native, mode, 0666); /* INTL: Native. */ Tcl_DStringFree(&ds); if (fd != -1) { fcntl(fd, F_SETFD, FD_CLOEXEC); @@ -196,11 +200,16 @@ fcntl(fd, F_SETFD, FD_CLOEXEC); if (contents != NULL) { Tcl_DString dstring; char *native; - native = Tcl_UtfToExternalDString(NULL, contents, TCL_INDEX_NONE, &dstring); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, contents, TCL_INDEX_NONE, 0, &dstring, NULL) != TCL_OK) { + close(fd); + Tcl_DStringFree(&dstring); + return NULL; + } + native = Tcl_DStringValue(&dstring); if (write(fd, native, Tcl_DStringLength(&dstring)) == -1) { close(fd); Tcl_DStringFree(&dstring); return NULL; } @@ -435,11 +444,19 @@ dsArray = (Tcl_DString *)TclStackAlloc(interp, argc * sizeof(Tcl_DString)); newArgv = (char **)TclStackAlloc(interp, (argc+1) * sizeof(char *)); newArgv[argc] = NULL; for (i = 0; i < argc; i++) { - newArgv[i] = Tcl_UtfToExternalDString(NULL, argv[i], TCL_INDEX_NONE, &dsArray[i]); + if (Tcl_UtfToExternalDStringEx(interp, NULL, argv[i], TCL_INDEX_NONE, 0, &dsArray[i], NULL) != TCL_OK) { + while (i-- > 0) { + Tcl_DStringFree(&dsArray[i]); + } + TclStackFree(interp, newArgv); + TclStackFree(interp, dsArray); + goto error; + } + newArgv[i] = Tcl_DStringValue(&dsArray[i]); } #ifdef USE_VFORK /* * After vfork(), do not call code in the child that changes global state, Index: win/tclWinPipe.c ================================================================== --- win/tclWinPipe.c +++ win/tclWinPipe.c @@ -649,11 +649,11 @@ TclFile TclpCreateTempFile( const char *contents) /* String to write into temp file, or NULL. */ { WCHAR name[MAX_PATH]; - const char *native; + const char *native = NULL; Tcl_DString dstring; HANDLE handle; if (TempFileName(name) == 0) { return NULL; @@ -677,11 +677,14 @@ /* * Convert the contents from UTF to native encoding */ - native = Tcl_UtfToExternalDString(NULL, contents, TCL_INDEX_NONE, &dstring); + if (Tcl_UtfToExternalDStringEx(NULL, NULL, contents, TCL_INDEX_NONE, 0, &dstring, NULL) != TCL_OK) { + goto error; + } + native = Tcl_DStringValue(&dstring); toCopy = Tcl_DStringLength(&dstring); for (p = native; toCopy > 0; p++, toCopy--) { if (*p == '\n') { length = p - native; @@ -717,11 +720,13 @@ if (contents != NULL) { Tcl_DStringFree(&dstring); } - Tcl_WinConvertError(GetLastError()); + if (native != NULL) { + Tcl_WinConvertError(GetLastError()); + } CloseHandle(handle); DeleteFileW(name); return NULL; }