Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch tip-656-simpl Through [8419a7023d] Excluding Merge-Ins
This is equivalent to a diff from e4a6023a61 to 8419a7023d
|
2023-04-14
| ||
| 10:27 | Merge 8.7 check-in: 8b54565968 user: jan.nijtmans tags: tip-656-simpl | |
|
2023-03-23
| ||
| 07:27 | fix for [f3cb2a32d6] Add initialization to allocation of string rep buffer to resolve valgrind repor... check-in: ca0e3d0976 user: jan.nijtmans tags: core-8-branch | |
|
2023-03-22
| ||
| 21:20 | Simplify TIP #656 implementation. This also changes the TCL_ENCODING_PROFILE_* macro's to the same v... check-in: 8419a7023d user: jan.nijtmans tags: tip-656-simpl | |
| 21:15 | Merge 9.0 check-in: f25757b233 user: jan.nijtmans tags: trunk, main | |
| 21:11 | Remove knownProfileBug constraint: this is already fixed check-in: e4a6023a61 user: jan.nijtmans tags: core-8-branch | |
| 20:09 | One missing int -> Tcl_Size change check-in: e84a92aa11 user: jan.nijtmans tags: core-8-branch | |
| 15:50 | In "replace" mode, "encoding convertfrom" can throw an exception as well if it doesn't receive a byt... check-in: 40c315b2b6 user: jan.nijtmans tags: tip-656-simpl | |
Changes to generic/tcl.h.
| ︙ | ︙ | |||
2129 2130 2131 2132 2133 2134 2135 | * NOTE: THESE BIT DEFINITIONS SHOULD NOT OVERLAP WITH INTERNAL USE BITS * DEFINED IN tclEncoding.c (ENCODING_INPUT et al). Be cognizant of this * when adding bits. */ #define TCL_ENCODING_START 0x01 #define TCL_ENCODING_END 0x02 | > > > | > | | | | | 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 | * NOTE: THESE BIT DEFINITIONS SHOULD NOT OVERLAP WITH INTERNAL USE BITS * DEFINED IN tclEncoding.c (ENCODING_INPUT et al). Be cognizant of this * when adding bits. */ #define TCL_ENCODING_START 0x01 #define TCL_ENCODING_END 0x02 #if TCL_MAJOR_VERSION > 8 # define TCL_ENCODING_STOPONERROR 0x0 /* Not used any more */ #else # define TCL_ENCODING_STOPONERROR 0x04 #endif #define TCL_ENCODING_NO_TERMINATE 0x08 #define TCL_ENCODING_CHAR_LIMIT 0x10 /* Internal use bits, do not define bits in this space. See above comment */ #define TCL_ENCODING_INTERNAL_USE_MASK 0xFF00 /* Reserve top byte for profile values (disjoint, not a mask) */ #define TCL_ENCODING_PROFILE_STRICT TCL_ENCODING_STOPONERROR #define TCL_ENCODING_PROFILE_TCL8 0x01000000 #define TCL_ENCODING_PROFILE_REPLACE 0x02000000 #define TCL_ENCODING_PROFILE_DEFAULT 0 /* * The following definitions are the error codes returned by the conversion * routines: * * TCL_OK - All characters were converted. * TCL_CONVERT_NOSPACE - The output buffer would not have been large |
| ︙ | ︙ |
Changes to generic/tclCmdAH.c.
| ︙ | ︙ | |||
671 672 673 674 675 676 677 |
return TCL_ERROR;
}
/*
* Convert the string into a byte array in 'ds'.
*/
#if !defined(TCL_NO_DEPRECATED) && (TCL_MAJOR_VERSION < 9)
| | > | 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 |
return TCL_ERROR;
}
/*
* Convert the string into a byte array in 'ds'.
*/
#if !defined(TCL_NO_DEPRECATED) && (TCL_MAJOR_VERSION < 9)
if (CHANNEL_PROFILE_GET(flags) != TCL_ENCODING_PROFILE_REPLACE
&& !(flags & TCL_ENCODING_STOPONERROR)) {
/* Permits high bits to be non-0 in byte array (Tcl 8 style) */
bytesPtr = (char *) Tcl_GetByteArrayFromObj(data, &length);
} else
#endif
bytesPtr = (char *) TclGetBytesFromObj(interp, data, &length);
if (bytesPtr == NULL) {
|
| ︙ | ︙ |
Changes to generic/tclEncoding.c.
| ︙ | ︙ | |||
196 197 198 199 200 201 202 |
int value;
} encodingProfiles[] = {
{"replace", TCL_ENCODING_PROFILE_REPLACE},
{"strict", TCL_ENCODING_PROFILE_STRICT},
{"tcl8", TCL_ENCODING_PROFILE_TCL8},
};
#define PROFILE_STRICT(flags_) \
| | > | 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 |
int value;
} encodingProfiles[] = {
{"replace", TCL_ENCODING_PROFILE_REPLACE},
{"strict", TCL_ENCODING_PROFILE_STRICT},
{"tcl8", TCL_ENCODING_PROFILE_TCL8},
};
#define PROFILE_STRICT(flags_) \
((CHANNEL_PROFILE_GET(flags_) == CHANNEL_PROFILE_STRICT) \
&& ((flags) && TCL_ENCODING_STOPONERROR))
#define PROFILE_REPLACE(flags_) \
(CHANNEL_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE)
#define UNICODE_REPLACE_CHAR ((Tcl_UniChar)0xFFFD)
#define SURROGATE(c_) (((c_) & ~0x7FF) == 0xD800)
#define HIGH_SURROGATE(c_) (((c_) & ~0x3FF) == 0xD800)
|
| ︙ | ︙ | |||
2443 2444 2445 2446 2447 2448 2449 |
int result;
result = TCL_OK;
dstLen -= TCL_UTF_MAX - 1;
if (dstLen < 0) {
dstLen = 0;
}
| < | 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 |
int result;
result = TCL_OK;
dstLen -= TCL_UTF_MAX - 1;
if (dstLen < 0) {
dstLen = 0;
}
if ((flags & TCL_ENCODING_CHAR_LIMIT) && srcLen > *dstCharsPtr) {
srcLen = *dstCharsPtr;
}
if (srcLen > dstLen) {
srcLen = dstLen;
result = TCL_CONVERT_NOSPACE;
}
|
| ︙ | ︙ | |||
2511 2512 2513 2514 2515 2516 2517 |
int profile;
result = TCL_OK;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
| < | 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 |
int profile;
result = TCL_OK;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
if ((flags & TCL_ENCODING_END) == 0) {
srcClose -= 6;
}
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
}
|
| ︙ | ︙ | |||
2727 2728 2729 2730 2731 2732 2733 |
* output buffer. */
{
const char *srcStart, *srcEnd;
const char *dstEnd, *dstStart;
int result, numChars, charLimit = INT_MAX;
int ch = 0, bytesLeft = srcLen % 4;
| < | 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 |
* output buffer. */
{
const char *srcStart, *srcEnd;
const char *dstEnd, *dstStart;
int result, numChars, charLimit = INT_MAX;
int ch = 0, bytesLeft = srcLen % 4;
flags |= PTR2INT(clientData);
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
}
result = TCL_OK;
/*
|
| ︙ | ︙ | |||
2884 2885 2886 2887 2888 2889 2890 |
const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
int result, numChars;
int ch, len;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
| < | 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 |
const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
int result, numChars;
int ch, len;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
if ((flags & TCL_ENCODING_END) == 0) {
srcClose -= TCL_UTF_MAX;
}
dstStart = dst;
dstEnd = dst + dstLen - sizeof(Tcl_UniChar);
flags |= PTR2INT(clientData);
|
| ︙ | ︙ | |||
2982 2983 2984 2985 2986 2987 2988 |
* output buffer. */
{
const char *srcStart, *srcEnd;
const char *dstEnd, *dstStart;
int result, numChars, charLimit = INT_MAX;
unsigned short ch = 0;
| < | 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 |
* output buffer. */
{
const char *srcStart, *srcEnd;
const char *dstEnd, *dstStart;
int result, numChars, charLimit = INT_MAX;
unsigned short ch = 0;
flags |= PTR2INT(clientData);
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
}
result = TCL_OK;
/*
|
| ︙ | ︙ | |||
3143 3144 3145 3146 3147 3148 3149 |
const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
int result, numChars;
int ch, len;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
| < | 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 |
const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
int result, numChars;
int ch, len;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
if ((flags & TCL_ENCODING_END) == 0) {
srcClose -= TCL_UTF_MAX;
}
dstStart = dst;
dstEnd = dst + dstLen - sizeof(Tcl_UniChar);
flags |= PTR2INT(clientData);
|
| ︙ | ︙ | |||
3249 3250 3251 3252 3253 3254 3255 |
* correspond to the bytes stored in the
* output buffer. */
{
const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
int result, numChars, len;
Tcl_UniChar ch = 0;
| < | 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 |
* correspond to the bytes stored in the
* output buffer. */
{
const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
int result, numChars, len;
Tcl_UniChar ch = 0;
flags |= PTR2INT(clientData);
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
if ((flags & TCL_ENCODING_END) == 0) {
srcClose -= TCL_UTF_MAX;
}
|
| ︙ | ︙ | |||
3372 3373 3374 3375 3376 3377 3378 |
const char *dstEnd, *dstStart, *prefixBytes;
int result, byte, numChars, charLimit = INT_MAX;
Tcl_UniChar ch = 0;
const unsigned short *const *toUnicode;
const unsigned short *pageZero;
TableEncodingData *dataPtr = (TableEncodingData *)clientData;
| < | 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 |
const char *dstEnd, *dstStart, *prefixBytes;
int result, byte, numChars, charLimit = INT_MAX;
Tcl_UniChar ch = 0;
const unsigned short *const *toUnicode;
const unsigned short *pageZero;
TableEncodingData *dataPtr = (TableEncodingData *)clientData;
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
}
srcStart = src;
srcEnd = src + srcLen;
dstStart = dst;
|
| ︙ | ︙ | |||
3506 3507 3508 3509 3510 3511 3512 |
prefixBytes = dataPtr->prefixBytes;
fromUnicode = (const unsigned short *const *) dataPtr->fromUnicode;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
| < | 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 |
prefixBytes = dataPtr->prefixBytes;
fromUnicode = (const unsigned short *const *) dataPtr->fromUnicode;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
if ((flags & TCL_ENCODING_END) == 0) {
srcClose -= TCL_UTF_MAX;
}
dstStart = dst;
dstEnd = dst + dstLen - 1;
|
| ︙ | ︙ | |||
3613 3614 3615 3616 3617 3618 3619 |
* correspond to the bytes stored in the
* output buffer. */
{
const char *srcStart, *srcEnd;
const char *dstEnd, *dstStart;
int result, numChars, charLimit = INT_MAX;
| < | 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 |
* correspond to the bytes stored in the
* output buffer. */
{
const char *srcStart, *srcEnd;
const char *dstEnd, *dstStart;
int result, numChars, charLimit = INT_MAX;
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
}
srcStart = src;
srcEnd = src + srcLen;
dstStart = dst;
|
| ︙ | ︙ | |||
3698 3699 3700 3701 3702 3703 3704 |
const char *dstStart, *dstEnd;
int result = TCL_OK, numChars;
Tcl_UniChar ch = 0;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
| < | 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 |
const char *dstStart, *dstEnd;
int result = TCL_OK, numChars;
Tcl_UniChar ch = 0;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
if ((flags & TCL_ENCODING_END) == 0) {
srcClose -= TCL_UTF_MAX;
}
dstStart = dst;
dstEnd = dst + dstLen - 1;
|
| ︙ | ︙ | |||
3846 3847 3848 3849 3850 3851 3852 |
EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
const char *prefixBytes, *tablePrefixBytes, *srcStart, *srcEnd;
const unsigned short *const *tableToUnicode;
const Encoding *encodingPtr;
int state, result, numChars, charLimit = INT_MAX;
const char *dstStart, *dstEnd;
| < | 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 |
EscapeEncodingData *dataPtr = (EscapeEncodingData *)clientData;
const char *prefixBytes, *tablePrefixBytes, *srcStart, *srcEnd;
const unsigned short *const *tableToUnicode;
const Encoding *encodingPtr;
int state, result, numChars, charLimit = INT_MAX;
const char *dstStart, *dstEnd;
if (flags & TCL_ENCODING_CHAR_LIMIT) {
charLimit = *dstCharsPtr;
}
result = TCL_OK;
tablePrefixBytes = NULL;
tableToUnicode = NULL;
prefixBytes = dataPtr->prefixBytes;
|
| ︙ | ︙ | |||
4070 4071 4072 4073 4074 4075 4076 |
Tcl_UniChar ch = 0;
result = TCL_OK;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
| < | 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 |
Tcl_UniChar ch = 0;
result = TCL_OK;
srcStart = src;
srcEnd = src + srcLen;
srcClose = srcEnd;
if ((flags & TCL_ENCODING_END) == 0) {
srcClose -= TCL_UTF_MAX;
}
dstStart = dst;
dstEnd = dst + dstLen - 1;
|
| ︙ | ︙ | |||
4515 4516 4517 4518 4519 4520 4521 |
"Internal error. Bad profile id \"%d\".",
profileValue));
Tcl_SetErrorCode(
interp, "TCL", "ENCODING", "PROFILEID", NULL);
}
return NULL;
}
| < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 |
"Internal error. Bad profile id \"%d\".",
profileValue));
Tcl_SetErrorCode(
interp, "TCL", "ENCODING", "PROFILEID", NULL);
}
return NULL;
}
/*
*------------------------------------------------------------------------
*
* TclGetEncodingProfiles --
*
* Get the list of supported encoding profiles.
|
| ︙ | ︙ |
Changes to generic/tclIO.c.
| ︙ | ︙ | |||
9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 |
* of the bytes themselves.
*/
/*
* TODO - should really only allow lossless profiles. Below reflects
* Tcl 8.7 alphas prior to encoding profiles
*/
moveBytes = inStatePtr->inEofChar == '\0' /* No eofChar to stop input */
&& inStatePtr->inputTranslation == TCL_TRANSLATE_LF
&& outStatePtr->outputTranslation == TCL_TRANSLATE_LF
&& inStatePtr->encoding == outStatePtr->encoding
| > > > > > > | | | 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 |
* of the bytes themselves.
*/
/*
* TODO - should really only allow lossless profiles. Below reflects
* Tcl 8.7 alphas prior to encoding profiles
*/
#define PROFILE_STRICT(flags_) \
((CHANNEL_PROFILE_GET(flags_) == CHANNEL_PROFILE_STRICT) \
&& ((flags_) && TCL_ENCODING_STOPONERROR))
#define PROFILE_REPLACE(flags_) \
(CHANNEL_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE)
moveBytes = inStatePtr->inEofChar == '\0' /* No eofChar to stop input */
&& inStatePtr->inputTranslation == TCL_TRANSLATE_LF
&& outStatePtr->outputTranslation == TCL_TRANSLATE_LF
&& inStatePtr->encoding == outStatePtr->encoding
&& (!PROFILE_STRICT(inStatePtr->flags) && !PROFILE_REPLACE(inStatePtr->flags))
&& (!PROFILE_STRICT(outStatePtr->flags) && !PROFILE_REPLACE(outStatePtr->flags));
/*
* Allocate a new CopyState to maintain info about the current copy in
* progress. This structure will be deallocated when the copy is
* completed.
*/
|
| ︙ | ︙ | |||
9793 9794 9795 9796 9797 9798 9799 |
* the copying. The caller uses Tcl_GetChannel to access it, and thus gets
* the bottom of the stack.
*/
inBinary = (inStatePtr->encoding == NULL);
outBinary = (outStatePtr->encoding == NULL);
sameEncoding = inStatePtr->encoding == outStatePtr->encoding
| | | | 9799 9800 9801 9802 9803 9804 9805 9806 9807 9808 9809 9810 9811 9812 9813 9814 |
* the copying. The caller uses Tcl_GetChannel to access it, and thus gets
* the bottom of the stack.
*/
inBinary = (inStatePtr->encoding == NULL);
outBinary = (outStatePtr->encoding == NULL);
sameEncoding = inStatePtr->encoding == outStatePtr->encoding
&& (!PROFILE_STRICT(inStatePtr->flags) && !PROFILE_REPLACE(inStatePtr->flags))
&& (!PROFILE_STRICT(outStatePtr->flags) && !PROFILE_REPLACE(outStatePtr->flags));
if (!(inBinary || sameEncoding)) {
TclNewObj(bufObj);
Tcl_IncrRefCount(bufObj);
}
while (csPtr->toRead != (Tcl_WideInt) 0) {
|
| ︙ | ︙ |
Changes to generic/tclIO.h.
| ︙ | ︙ | |||
283 284 285 286 287 288 289 290 291 292 293 294 295 296 |
* usable, but it may not be closed
* again from within the close
* handler. */
#define CHANNEL_CLOSEDWRITE (1<<21) /* Channel write side has been closed.
* No further Tcl-level write IO on
* the channel is allowed. */
#define CHANNEL_PROFILE_MASK 0xFF000000
#define CHANNEL_PROFILE_GET(flags_) ((flags_) & CHANNEL_PROFILE_MASK)
#define CHANNEL_PROFILE_SET(flags_, profile_) \
do { \
(flags_) &= ~CHANNEL_PROFILE_MASK; \
(flags_) |= profile_; \
} while (0)
| > | 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 |
* usable, but it may not be closed
* again from within the close
* handler. */
#define CHANNEL_CLOSEDWRITE (1<<21) /* Channel write side has been closed.
* No further Tcl-level write IO on
* the channel is allowed. */
#define CHANNEL_PROFILE_MASK 0xFF000000
#define CHANNEL_PROFILE_STRICT (TCL_ENCODING_PROFILE_STRICT & CHANNEL_PROFILE_MASK)
#define CHANNEL_PROFILE_GET(flags_) ((flags_) & CHANNEL_PROFILE_MASK)
#define CHANNEL_PROFILE_SET(flags_, profile_) \
do { \
(flags_) &= ~CHANNEL_PROFILE_MASK; \
(flags_) |= profile_; \
} while (0)
|
| ︙ | ︙ |
Changes to generic/tclInt.h.
| ︙ | ︙ | |||
2886 2887 2888 2889 2890 2891 2892 | MODULE_SCOPE Tcl_Encoding tclIdentityEncoding; MODULE_SCOPE int TclEncodingProfileNameToId(Tcl_Interp *interp, const char *profileName, int *profilePtr); MODULE_SCOPE const char *TclEncodingProfileIdToName(Tcl_Interp *interp, int profileId); | < | 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 | MODULE_SCOPE Tcl_Encoding tclIdentityEncoding; MODULE_SCOPE int TclEncodingProfileNameToId(Tcl_Interp *interp, const char *profileName, int *profilePtr); MODULE_SCOPE const char *TclEncodingProfileIdToName(Tcl_Interp *interp, int profileId); MODULE_SCOPE void TclGetEncodingProfiles(Tcl_Interp *interp); /* * TIP #233 (Virtualized Time) * Data for the time hooks, if any. */ |
| ︙ | ︙ |