| ︙ | | | ︙ | |
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
#if TCL_UTF_MAX > 3
4,4,4,4,4,
#else
1,1,1,1,1,
#endif
1,1,1,1,1,1,1,1,1,1,1
};
/*
*---------------------------------------------------------------------------
*
* TclUtfCount --
*
* Find the number of bytes in the Utf character "ch".
|
>
>
>
>
>
>
|
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
#if TCL_UTF_MAX > 3
4,4,4,4,4,
#else
1,1,1,1,1,
#endif
1,1,1,1,1,1,1,1,1,1,1
};
/*
* Functions used only in this module.
*/
static int Invalid(unsigned char *src);
/*
*---------------------------------------------------------------------------
*
* TclUtfCount --
*
* Find the number of bytes in the Utf character "ch".
|
| ︙ | | | ︙ | |
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
return 2;
}
if (((unsigned)(ch - 0x10000) <= 0xFFFFF)) {
return 4;
}
return 3;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UniCharToUtf --
*
* Store the given Tcl_UniChar as a sequence of UTF-8 bytes in the
* provided buffer. Equivalent to Plan 9 runetochar().
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
return 2;
}
if (((unsigned)(ch - 0x10000) <= 0xFFFFF)) {
return 4;
}
return 3;
}
/*
*---------------------------------------------------------------------------
*
* Invalid --
*
* Utility routine to report whether /src/ points to the start of an
* invald byte sequence that should be rejected. This might be because
* it is an overlong encoding, or because it encodes something out of
* the proper range. Caller guarantees that src[0] and src[1] are
* readable, and
*
* (src[0] >= 0xC0) && (src[0] != 0xC1)
* (src[1] >= 0x80) && (src[1] < 0xC0)
* (src[0] < ((TCL_UTF_MAX > 3) ? 0xF5 : 0xF0))
*
* Results:
* A boolean.
*---------------------------------------------------------------------------
*/
static const unsigned char bounds[28] = {
0x80, 0x80, /* \xC0 accepts \x80 only */
0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF,
0x80, 0xBF, /* (\xC4 - \xDC) -- all sequences valid */
0xA0, 0xBF, /* \xE0\x80 through \xE0\x9F are invalid prefixes */
0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF, /* (\xE4 - \xEC) -- all valid */
0x90, 0xBF, /* \xF0\x80 through \xF0\x8F are invalid prefixes */
0x80, 0x8F /* \xF4\x90 and higher are invalid prefixes */
};
static int
Invalid(
unsigned char *src) /* Points to lead byte of a UTF-8 byte sequence */
{
unsigned char byte = *src;
int index;
if (byte % 0x04) {
/* Only lead bytes 0xC0, 0xE0, 0xF0, 0xF4 need examination */
return 0;
}
index = (byte - 0xC0) >> 1;
if (src[1] < bounds[index] || src[1] > bounds[index+1]) {
/* Out of bounds - report invalid. */
return 1;
}
return 0;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UniCharToUtf --
*
* Store the given Tcl_UniChar as a sequence of UTF-8 bytes in the
* provided buffer. Equivalent to Plan 9 runetochar().
|
| ︙ | | | ︙ | |
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
|
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UtfNext --
*
* Given a pointer to some location in a UTF-8 string, Tcl_UtfNext
* returns a pointer to the next UTF-8 character in the string.
* The caller must not ask for the next character after the last
* character in the string if the string is not terminated by a null
* character.
*
* Results:
* The return value is the pointer to the next character in the UTF-8
* string.
*
* Side effects:
* None.
*
*---------------------------------------------------------------------------
*/
const char *
Tcl_UtfNext(
const char *src) /* The current location in the string. */
{
Tcl_UniChar ch = 0;
size_t len;
if (((*src) & 0xC0) == 0x80) {
if ((((*++src) & 0xC0) == 0x80) && (((*++src) & 0xC0) == 0x80)) {
++src;
}
return src;
}
len = TclUtfToUniChar(src, &ch);
#if TCL_UTF_MAX <= 3
if ((ch >= 0xD800) && (len < 3)) {
len += TclUtfToUniChar(src + len, &ch);
}
#endif
return src + len;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UtfPrev --
*
|
|
<
|
>
<
<
>
|
>
>
>
>
>
|
|
|
>
>
|
>
>
|
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
|
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UtfNext --
*
* Given a pointer to some location in a UTF-8 string, Tcl_UtfNext
* returns a pointer to the next UTF-8 character in the string.
* The caller must not ask for the next character after the last
* character in the string if the string is not terminated by a null
* character.
*
* Results:
* The return value is the pointer to the next character in the UTF-8
* string.
*
* Side effects:
* None.
*
*---------------------------------------------------------------------------
*/
const char *
Tcl_UtfNext(
const char *src) /* The current location in the string. */
{
size_t left = totalBytes[UCHAR(*src)];
const char *next = src + 1;
if (((*src) & 0xC0) == 0x80) {
if ((((*++src) & 0xC0) == 0x80) && (((*++src) & 0xC0) == 0x80)) {
++src;
}
return src;
}
while (--left) {
if ((*next & 0xC0) != 0x80) {
/*
* src points to non-trail byte; We ran out of trail bytes
* before the needs of the lead byte were satisfied.
* Let the (malformed) lead byte alone be a character
*/
return src + 1;
}
next++;
}
if (Invalid((unsigned char *)src)) {
return src + 1;
}
return next;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UtfPrev --
*
|
| ︙ | | | ︙ | |
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
|
*/
const char *
Tcl_UtfPrev(
const char *src, /* A location in a UTF-8 string. */
const char *start) /* Pointer to the beginning of the string */
{
const char *look;
int i, byte;
look = --src;
for (i = 0; i < 4; i++) {
if (look < start) {
if (src < start) {
src = start;
}
break;
}
byte = *((unsigned char *) look);
if (byte < 0x80) {
break;
}
if (byte >= 0xC0) {
if (totalBytes[byte] <= i) {
break;
}
return look;
}
look--;
}
return src;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UniCharAtIndex --
*
|
>
|
|
>
>
>
>
<
<
>
|
<
|
|
<
|
>
|
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
>
>
>
|
>
>
>
>
>
>
|
|
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
|
*/
const char *
Tcl_UtfPrev(
const char *src, /* A location in a UTF-8 string. */
const char *start) /* Pointer to the beginning of the string */
{
int trailBytesSeen = 0; /* How many trail bytes have been verified? */
const char *fallback = src - 1;
/* If we cannot find a lead byte that might
* start a prefix of a valid UTF byte sequence,
* we will fallback to a one-byte back step */
unsigned char *look = (unsigned char *)fallback;
/* Start search at the fallback position */
/* Quick boundary case exit. */
if (fallback <= start) {
return start;
}
do {
unsigned char byte = look[0];
if (byte < 0x80) {
/*
* Single byte character. Either this is a correct previous
* character, or it is followed by at least one trail byte
* which indicates a malformed sequence. In either case the
* correct result is to return the fallback.
*/
return fallback;
}
if (byte >= 0xC0) {
/* Non-trail byte; May be multibyte lead. */
if ((trailBytesSeen == 0)
/*
* We've seen no trailing context to use to check
* anything. From what we know, this non-trail byte
* is a prefix of a previous character, and accepting
* it (the fallback) is correct.
*/
|| (trailBytesSeen >= totalBytes[byte])) {
/*
* That is, (1 + trailBytesSeen > needed).
* We've examined more bytes than needed to complete
* this lead byte. No matter about well-formedness or
* validity, the sequence starting with this lead byte
* will never include the fallback location, so we must
* return the fallback location. See test utf-7.17
*/
return fallback;
}
/*
* trailBytesSeen > 0, so we can examine look[1] safely.
* Use that capability to screen out overlong sequences.
*/
if (Invalid(look)) {
/* Reject */
return fallback;
}
return (const char *)look;
}
/* We saw a trail byte. */
trailBytesSeen++;
if ((const char *)look == start) {
/*
* Do not read before the start of the string
*
* If we get here, we've examined bytes at every location
* >= start and < src and all of them are trail bytes,
* including (*start). We need to return our fallback
* and exit this loop before we run past the start of the string.
*/
return fallback;
}
/* Continue the search backwards... */
look--;
} while (trailBytesSeen < 4);
/*
* We've seen TCL_UTF_MAX trail bytes, so we know there will not be a
* properly formed byte sequence to find, and we can stop looking,
* accepting the fallback (for TCL_UTF_MAX > 3) or just go back as
* far as we can.
*/
return fallback;
}
/*
*---------------------------------------------------------------------------
*
* Tcl_UniCharAtIndex --
*
|
| ︙ | | | ︙ | |