Differences From Artifact [d282366dbc]:
- File generic/tclUtf.c — part of check-in [c59e41ca4e] at 2020-05-07 21:28:19 on branch bug-ed29806baf-8.6 — Merge 8.6 (user: jan.nijtmans size: 61589) [more...]
To Artifact [bff485763b]:
- File generic/tclUtf.c — part of check-in [aeb46f9ed0] at 2020-05-11 12:25:03 on branch bug-ed29806baf-8.6 — Merge 8.6. Mark testcase utf-4.11 as "knownBug": this one still doesn't give the right answer. Add testcase 4.14 with similar corner-case, this one is OK. (user: jan.nijtmans size: 62194) [more...]
| ︙ | ︙ | |||
60 61 62 63 64 65 66 |
*/
static const unsigned char totalBytes[256] = {
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
| > > > | > > > > > > > > > > | > > > > > > > > > > > > | > > | 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 |
*/
static const unsigned char totalBytes[256] = {
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
#if TCL_UTF_MAX < 4
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
#else /* Tcl_UtfCharComplete() might point to 2nd byte of valid 4-byte sequence */
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#endif
2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#if TCL_UTF_MAX > 4
4,4,4,4,4,
#else
1,1,1,1,1,
#endif
1,1,1,1,1,1,1,1,1,1,1
};
static const unsigned char complete[256] = {
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
#if TCL_UTF_MAX < 4
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
#else /* Tcl_UtfCharComplete() might point to 2nd byte of valid 4-byte sequence */
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#endif
2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
#if TCL_UTF_MAX > 4
4,4,4,4,4,
#elif TCL_UTF_MAX < 4
1,1,1,1,1,
#else
3,3,3,3,3,
#endif
1,1,1,1,1,1,1,1,1,1,1
};
/*
* Functions used only in this module.
*/
|
| ︙ | ︙ | |||
167 168 169 170 171 172 173 |
static int
Invalid(
const char *src) /* Points to lead byte of a UTF-8 byte sequence */
{
unsigned char byte = UCHAR(*src);
int index;
| | < < | | | | > | 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 |
static int
Invalid(
const char *src) /* Points to lead byte of a UTF-8 byte sequence */
{
unsigned char byte = UCHAR(*src);
int index;
if ((byte & 0xC3) == 0xC0) {
/* Only lead bytes 0xC0, 0xE0, 0xF0, 0xF4 need examination */
index = (byte - 0xC0) >> 1;
if (UCHAR(src[1]) < bounds[index] || UCHAR(src[1]) > bounds[index+1]) {
/* Out of bounds - report invalid. */
return 1;
}
}
return 0;
}
/*
*---------------------------------------------------------------------------
*
|
| ︙ | ︙ | |||
561 562 563 564 565 566 567 |
int
Tcl_UtfCharComplete(
const char *src, /* String to check if first few bytes contain
* a complete UTF-8 character. */
int length) /* Length of above string in bytes. */
{
| | | 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 |
int
Tcl_UtfCharComplete(
const char *src, /* String to check if first few bytes contain
* a complete UTF-8 character. */
int length) /* Length of above string in bytes. */
{
return length >= complete[UCHAR(*src)];
}
/*
*---------------------------------------------------------------------------
*
* Tcl_NumUtfChars --
*
|
| ︙ | ︙ | |||
614 615 616 617 618 619 620 |
* when (src <= optPtr), (endPtr - src) >= (endPtr - optPtr)
* By initialization above (endPtr - optPtr) = TCL_UTF_MAX
* So (endPtr - src) >= TCL_UTF_MAX, and passing that to
* Tcl_UtfCharComplete we know will cause return of 1.
*/
while (src <= optPtr
/* && Tcl_UtfCharComplete(src, endPtr - src) */ ) {
| < < < < < < < < < < < < < < | | | 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 |
* when (src <= optPtr), (endPtr - src) >= (endPtr - optPtr)
* By initialization above (endPtr - optPtr) = TCL_UTF_MAX
* So (endPtr - src) >= TCL_UTF_MAX, and passing that to
* Tcl_UtfCharComplete we know will cause return of 1.
*/
while (src <= optPtr
/* && Tcl_UtfCharComplete(src, endPtr - src) */ ) {
src += TclUtfToUniChar(src, &ch);
i++;
}
/* Loop over the remaining string where call must happen */
while (src < endPtr) {
if (Tcl_UtfCharComplete(src, endPtr - src)) {
src += TclUtfToUniChar(src, &ch);
} else {
/*
* src points to incomplete UTF-8 sequence
* Treat first byte as character and count it
*/
src++;
}
i++;
}
}
return i;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UtfFindFirst --
*
* Returns a pointer to the first occurrence of the given Unicode character
* in the NULL-terminated UTF-8 string. The NULL terminator is considered
* part of the UTF-8 string. Equivalent to Plan 9 utfrune().
*
* Results:
* As above. If the Unicode character does not exist in the given string,
* the return value is NULL.
*
|
| ︙ | ︙ | |||
690 691 692 693 694 695 696 | } /* *--------------------------------------------------------------------------- * * Tcl_UtfFindLast -- * | | | 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 | } /* *--------------------------------------------------------------------------- * * Tcl_UtfFindLast -- * * Returns a pointer to the last occurrence of the given Unicode character * in the NULL-terminated UTF-8 string. The NULL terminator is considered * part of the UTF-8 string. Equivalent to Plan 9 utfrrune(). * * Results: * As above. If the Unicode character does not exist in the given string, the * return value is NULL. * |
| ︙ | ︙ | |||
841 842 843 844 845 846 847 | /* * We've seen no trailing context to use to check * anything. From what we know, this non-trail byte * is a prefix of a previous character, and accepting * it (the fallback) is correct. */ | | | 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 |
/*
* We've seen no trailing context to use to check
* anything. From what we know, this non-trail byte
* is a prefix of a previous character, and accepting
* it (the fallback) is correct.
*/
|| (trailBytesSeen >= complete[byte])) {
/*
* That is, (1 + trailBytesSeen > needed).
* We've examined more bytes than needed to complete
* this lead byte. No matter about well-formedness or
* validity, the sequence starting with this lead byte
* will never include the fallback location, so we must
* return the fallback location. See test utf-7.17
|
| ︙ | ︙ | |||
1057 1058 1059 1060 1061 1062 1063 | /* * To keep badly formed Utf strings from getting inflated by the * conversion (thereby causing a segfault), only copy the upper case * char to dst if its size is <= the original char. */ | | | | 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 |
/*
* To keep badly formed Utf strings from getting inflated by the
* conversion (thereby causing a segfault), only copy the upper case
* char to dst if its size is <= the original char.
*/
if (len < UtfCount(upChar)) {
memmove(dst, src, len);
dst += len;
} else {
dst += TclUCS4ToUtf(upChar, dst);
}
src += len;
}
*dst = '\0';
return (dst - str);
}
|
| ︙ | ︙ | |||
1110 1111 1112 1113 1114 1115 1116 | /* * To keep badly formed Utf strings from getting inflated by the * conversion (thereby causing a segfault), only copy the lower case * char to dst if its size is <= the original char. */ | | | | 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 |
/*
* To keep badly formed Utf strings from getting inflated by the
* conversion (thereby causing a segfault), only copy the lower case
* char to dst if its size is <= the original char.
*/
if (len < UtfCount(lowChar)) {
memmove(dst, src, len);
dst += len;
} else {
dst += TclUCS4ToUtf(lowChar, dst);
}
src += len;
}
*dst = '\0';
return (dst - str);
}
|
| ︙ | ︙ | |||
1160 1161 1162 1163 1164 1165 1166 |
src = dst = str;
if (*src) {
len = TclUtfToUCS4(src, &ch);
titleChar = UCS4ToTitle(ch);
| | | | | | 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 |
src = dst = str;
if (*src) {
len = TclUtfToUCS4(src, &ch);
titleChar = UCS4ToTitle(ch);
if (len < UtfCount(titleChar)) {
memmove(dst, src, len);
dst += len;
} else {
dst += TclUCS4ToUtf(titleChar, dst);
}
src += len;
}
while (*src) {
len = TclUtfToUCS4(src, &ch);
lowChar = ch;
/* Special exception for Georgian Asomtavruli chars, no titlecase. */
if ((unsigned)(lowChar - 0x1C90) >= 0x30) {
lowChar = TclUCS4ToLower(lowChar);
}
if (len < UtfCount(lowChar)) {
memmove(dst, src, len);
dst += len;
} else {
dst += TclUCS4ToUtf(lowChar, dst);
}
src += len;
}
*dst = '\0';
return (dst - str);
}
|
| ︙ | ︙ |