Differences From Artifact [09f91effbd]:
- File generic/tclUtf.c — part of check-in [e2278643dc] at 2015-09-01 18:54:58 on branch trunk — Various Unicode handling enhancements, when building with TCL_UTF_MAX > 3, inspired by androwish. No effect if TCL_UTF_MAX=3 (which is the default) (user: jan.nijtmans size: 47967) [more...]
To Artifact [f38d3847d1]:
- File generic/tclUtf.c — part of check-in [c0a65532a7] at 2016-08-30 13:00:32 on branch core-8-6-branch — Don't ever allow UTF-8 sequences of more than 4 characters to be generated or parsed, even when TCL_UTF_MAX>4: According to current Unicode standard, a byte string of >4 characters can never form a single UTF-8 character. And a few minor micro-optimizations related to UTF-8 handling. (user: jan.nijtmans size: 47926) [more...]
| ︙ | ︙ | |||
69 70 71 72 73 74 75 |
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#if TCL_UTF_MAX > 3
4,4,4,4,4,4,4,4,
#else
1,1,1,1,1,1,1,1,
#endif
| < < < | < < < < < < | 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#if TCL_UTF_MAX > 3
4,4,4,4,4,4,4,4,
#else
1,1,1,1,1,1,1,1,
#endif
1,1,1,1,1,1,1,1
};
/*
* Functions used only in this module.
*/
static int UtfCount(int ch);
|
| ︙ | ︙ | |||
107 108 109 110 111 112 113 |
*---------------------------------------------------------------------------
*/
INLINE static int
UtfCount(
int ch) /* The Tcl_UniChar whose size is returned. */
{
| | | | 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 |
*---------------------------------------------------------------------------
*/
INLINE static int
UtfCount(
int ch) /* The Tcl_UniChar whose size is returned. */
{
if ((unsigned)(ch - 1) < (UNICODE_SELF - 1)) {
return 1;
}
if (ch <= 0x7FF) {
return 2;
}
#if TCL_UTF_MAX > 3
if (((unsigned)(ch - 0x10000) <= 0xfffff)) {
return 4;
}
#endif
return 3;
}
/*
|
| ︙ | ︙ | |||
148 149 150 151 152 153 154 |
int ch, /* The Tcl_UniChar to be stored in the
* buffer. */
char *buf) /* Buffer in which the UTF-8 representation of
* the Tcl_UniChar is stored. Buffer must be
* large enough to hold the UTF-8 character
* (at most TCL_UTF_MAX bytes). */
{
| | | 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 |
int ch, /* The Tcl_UniChar to be stored in the
* buffer. */
char *buf) /* Buffer in which the UTF-8 representation of
* the Tcl_UniChar is stored. Buffer must be
* large enough to hold the UTF-8 character
* (at most TCL_UTF_MAX bytes). */
{
if ((unsigned)(ch - 1) < (UNICODE_SELF - 1)) {
buf[0] = (char) ch;
return 1;
}
if (ch >= 0) {
if (ch <= 0x7FF) {
buf[1] = (char) ((ch | 0x80) & 0xBF);
buf[0] = (char) ((ch >> 6) | 0xC0);
|
| ︙ | ︙ | |||
176 177 178 179 180 181 182 | buf[2] = (char) (((ch << 4) | 0x80) & 0xB0); buf[1] = (char) (((ch >> 2) | 0x80) & 0xBF); buf[0] = (char) (((ch >> 8) | 0xF0) & 0xF7); return 0; } } #endif | | < < < < | > > > > | 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
buf[2] = (char) (((ch << 4) | 0x80) & 0xB0);
buf[1] = (char) (((ch >> 2) | 0x80) & 0xBF);
buf[0] = (char) (((ch >> 8) | 0xF0) & 0xF7);
return 0;
}
}
#endif
goto three;
}
#if TCL_UTF_MAX > 3
if (ch <= 0x10FFFF) {
buf[3] = (char) ((ch | 0x80) & 0xBF);
buf[2] = (char) (((ch >> 6) | 0x80) & 0xBF);
buf[1] = (char) (((ch >> 12) | 0x80) & 0xBF);
buf[0] = (char) ((ch >> 18) | 0xF0);
return 4;
}
#endif
}
ch = 0xFFFD;
three:
buf[2] = (char) ((ch | 0x80) & 0xBF);
buf[1] = (char) (((ch >> 6) | 0x80) & 0xBF);
buf[0] = (char) ((ch >> 12) | 0xE0);
return 3;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UniCharToUtfDString --
*
|
| ︙ | ︙ | |||
310 311 312 313 314 315 316 | return 2; } /* * A two-byte-character lead-byte not followed by trail-byte * represents itself. */ | < < < | < < < > > > > > | < | < < < < < | < < | | < < < < < < | > > > > | 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 |
return 2;
}
/*
* A two-byte-character lead-byte not followed by trail-byte
* represents itself.
*/
} else if (byte < 0xF0) {
if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80)) {
/*
* Three-byte-character lead byte followed by two trail bytes.
*/
*chPtr = (Tcl_UniChar) (((byte & 0x0F) << 12)
| ((src[1] & 0x3F) << 6) | (src[2] & 0x3F));
return 3;
}
/*
* A three-byte-character lead-byte not followed by two trail-bytes
* represents itself.
*/
}
#if TCL_UTF_MAX > 3
else if (byte < 0xF8) {
if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80) && ((src[3] & 0xC0) == 0x80)) {
/*
* Four-byte-character lead byte followed by three trail bytes.
*/
*chPtr = (Tcl_UniChar) (((byte & 0x0E) << 18) | ((src[1] & 0x3F) << 12)
| ((src[2] & 0x3F) << 6) | (src[3] & 0x3F));
return 4;
}
/*
* A three-byte-character lead-byte not followed by two trail-bytes
* represents itself.
*/
}
#endif
*chPtr = (Tcl_UniChar) byte;
return 1;
}
|
| ︙ | ︙ |