Differences From Artifact [c78a847600]:
- File generic/tclUtf.c — part of check-in [334ab96e5e] at 2013-07-29 09:29:16 on branch core-8-5-branch — Make sure that "string is space \u202f" will continue to return "1", even if in future Unicode this character (NARROW_NO_BREAK_SPACE) will cease to be a space. See: [http://www.unicode.org/review/pri249/] (user: jan.nijtmans size: 46902)
To Artifact [3a98891648]:
- File generic/tclUtf.c — part of check-in [3998688718] at 2017-06-06 09:02:28 on branch core-8-5-branch — [67aa9a2070] Tcl_UtfToUniChar returns single byte for invalid UTF-8 input as documented. (user: jan.nijtmans size: 46746)
| ︙ | ︙ | |||
69 70 71 72 73 74 75 |
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#if TCL_UTF_MAX > 3
4,4,4,4,4,4,4,4,
#else
1,1,1,1,1,1,1,1,
#endif
| < < < | < < < < < < | 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#if TCL_UTF_MAX > 3
4,4,4,4,4,4,4,4,
#else
1,1,1,1,1,1,1,1,
#endif
1,1,1,1,1,1,1,1
};
/*
* Functions used only in this module.
*/
static int UtfCount(int ch);
|
| ︙ | ︙ | |||
107 108 109 110 111 112 113 |
*---------------------------------------------------------------------------
*/
INLINE static int
UtfCount(
int ch) /* The Tcl_UniChar whose size is returned. */
{
| | < < < | < < < < < < | 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
*---------------------------------------------------------------------------
*/
INLINE static int
UtfCount(
int ch) /* The Tcl_UniChar whose size is returned. */
{
if ((unsigned)(ch - 1) < (UNICODE_SELF - 1)) {
return 1;
}
if (ch <= 0x7FF) {
return 2;
}
#if TCL_UTF_MAX > 3
if (((unsigned)(ch - 0x10000) <= 0xFFFFF)) {
return 4;
}
#endif
return 3;
}
/*
*---------------------------------------------------------------------------
*
|
| ︙ | ︙ | |||
157 158 159 160 161 162 163 |
int ch, /* The Tcl_UniChar to be stored in the
* buffer. */
char *buf) /* Buffer in which the UTF-8 representation of
* the Tcl_UniChar is stored. Buffer must be
* large enough to hold the UTF-8 character
* (at most TCL_UTF_MAX bytes). */
{
| | < > > > > | > > > > > > | | | | | > > > > | < < < < < < < < < < < < < < < < < | > > > > | 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
int ch, /* The Tcl_UniChar to be stored in the
* buffer. */
char *buf) /* Buffer in which the UTF-8 representation of
* the Tcl_UniChar is stored. Buffer must be
* large enough to hold the UTF-8 character
* (at most TCL_UTF_MAX bytes). */
{
if ((unsigned)(ch - 1) < (UNICODE_SELF - 1)) {
buf[0] = (char) ch;
return 1;
}
if (ch >= 0) {
if (ch <= 0x7FF) {
buf[1] = (char) ((ch | 0x80) & 0xBF);
buf[0] = (char) ((ch >> 6) | 0xC0);
return 2;
}
if (ch <= 0xFFFF) {
#if TCL_UTF_MAX == 4
if ((ch & 0xF800) == 0xD800) {
if (ch & 0x0400) {
/* Low surrogate */
buf[3] = (char) ((ch | 0x80) & 0xBF);
buf[2] |= (char) (((ch >> 6) | 0x80) & 0x8F);
return 4;
} else {
/* High surrogate */
ch += 0x40;
buf[2] = (char) (((ch << 4) | 0x80) & 0xB0);
buf[1] = (char) (((ch >> 2) | 0x80) & 0xBF);
buf[0] = (char) (((ch >> 8) | 0xF0) & 0xF7);
return 0;
}
}
#endif
goto three;
}
#if TCL_UTF_MAX > 3
if (ch <= 0x10FFFF) {
buf[3] = (char) ((ch | 0x80) & 0xBF);
buf[2] = (char) (((ch >> 6) | 0x80) & 0xBF);
buf[1] = (char) (((ch >> 12) | 0x80) & 0xBF);
buf[0] = (char) ((ch >> 18) | 0xF0);
return 4;
}
#endif
}
ch = 0xFFFD;
three:
buf[2] = (char) ((ch | 0x80) & 0xBF);
buf[1] = (char) (((ch >> 6) | 0x80) & 0xBF);
buf[0] = (char) ((ch >> 12) | 0xE0);
return 3;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UniCharToUtfDString --
*
|
| ︙ | ︙ | |||
312 313 314 315 316 317 318 |
} else if (byte < 0xE0) {
if ((src[1] & 0xC0) == 0x80) {
/*
* Two-byte-character lead-byte followed by a trail-byte.
*/
*chPtr = (Tcl_UniChar) (((byte & 0x1F) << 6) | (src[1] & 0x3F));
| > | > < < < > | > | < < < > > > > > | < | < < < < < | < | | | < < < < < < > > > > > | 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 |
} else if (byte < 0xE0) {
if ((src[1] & 0xC0) == 0x80) {
/*
* Two-byte-character lead-byte followed by a trail-byte.
*/
*chPtr = (Tcl_UniChar) (((byte & 0x1F) << 6) | (src[1] & 0x3F));
if ((unsigned)(*chPtr - 1) >= (UNICODE_SELF - 1)) {
return 2;
}
}
/*
* A two-byte-character lead-byte not followed by trail-byte
* represents itself.
*/
} else if (byte < 0xF0) {
if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80)) {
/*
* Three-byte-character lead byte followed by two trail bytes.
*/
*chPtr = (Tcl_UniChar) (((byte & 0x0F) << 12)
| ((src[1] & 0x3F) << 6) | (src[2] & 0x3F));
if (*chPtr > 0x7ff) {
return 3;
}
}
/*
* A three-byte-character lead-byte not followed by two trail-bytes
* represents itself.
*/
}
#if TCL_UTF_MAX > 3
else if (byte < 0xF8) {
if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80) && ((src[3] & 0xC0) == 0x80)) {
/*
* Four-byte-character lead byte followed by three trail bytes.
*/
*chPtr = (Tcl_UniChar) (((byte & 0x07) << 18) | ((src[1] & 0x3F) << 12)
| ((src[2] & 0x3F) << 6) | (src[3] & 0x3F));
if ((unsigned)(*chPtr - 0x10000) <= 0xFFFFF) {
return 4;
}
}
/*
* A four-byte-character lead-byte not followed by two trail-bytes
* represents itself.
*/
}
#endif
*chPtr = (Tcl_UniChar) byte;
return 1;
}
|
| ︙ | ︙ | |||
1034 1035 1036 1037 1038 1039 1040 |
CONST char *ct, /* UTF string cs is compared to. */
unsigned long numChars) /* Number of UTF chars to compare. */
{
Tcl_UniChar ch1, ch2;
/*
* Cannot use 'memcmp(cs, ct, n);' as byte representation of \u0000 (the
| | | 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 |
CONST char *ct, /* UTF string cs is compared to. */
unsigned long numChars) /* Number of UTF chars to compare. */
{
Tcl_UniChar ch1, ch2;
/*
* Cannot use 'memcmp(cs, ct, n);' as byte representation of \u0000 (the
* pair of bytes 0xC0,0x80) is larger than byte representation of \u0001
* (the byte 0x01.)
*/
while (numChars-- > 0) {
/*
* n must be interpreted as chars, not bytes. This should be called
* only when both strings are of at least n chars long (no need for \0
|
| ︙ | ︙ | |||
1551 1552 1553 1554 1555 1556 1557 |
/*
* If the character is within the first 127 characters, just use the
* standard C function, otherwise consult the Unicode table.
*/
if (((Tcl_UniChar) ch) < ((Tcl_UniChar) 0x80)) {
return TclIsSpaceProc((char) ch);
| | | 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 |
/*
* If the character is within the first 127 characters, just use the
* standard C function, otherwise consult the Unicode table.
*/
if (((Tcl_UniChar) ch) < ((Tcl_UniChar) 0x80)) {
return TclIsSpaceProc((char) ch);
} else if ((Tcl_UniChar) ch == 0x180E || (Tcl_UniChar) ch == 0x202F) {
return 1;
} else {
return ((SPACE_BITS >> GetCategory(ch)) & 1);
}
}
/*
|
| ︙ | ︙ |