Index: doc/Utf.3 ================================================================== --- doc/Utf.3 +++ doc/Utf.3 @@ -251,15 +251,24 @@ \fBTcl_UtfFindLast\fR corresponds to \fBstrrchr\fR for UTF-8 strings. It returns a pointer to the last occurrence of the Unicode character \fIch\fR in the null-terminated UTF-8 string \fIsrc\fR. The null terminator is considered part of the UTF-8 string. .PP -Given \fIsrc\fR, a pointer to some location in a UTF-8 string, -\fBTcl_UtfNext\fR returns a pointer to the next UTF-8 character in the -string. The caller must not ask for the next character after the last -character in the string if the string is not terminated by a null -character. +\fBTcl_UtfNext\fR is used to step forward through a UTF-8 string. +If the UTF-8 string is made up entirely of complete, well-formed, and +valid character byte sequences, and \fIsrc\fR points to the lead byte +of one of those sequences, then repeated calls of \fBTcl_UtfNext\fR will +return pointers to the lead bytes of each character in the string, one +character at a time. In any other circumstance, \fBTcl_UtfNext\fR +returns \fIsrc\fR+1. \fBTcl_UtfNext\fR will always read \fIsrc[0]\fR +and may read as many following bytes (up to a total of \fBTCL_UTF_MAX\fR) +as needed to find the end of the byte sequence. If the string is +\fBNUL\fR-terminated, \fBTcl_UtfNext\fR will not read beyond the terminating +\fBNUL\fR byte. If not, the caller must use the companion routine +\fBTcl_UtfCharComplete\fR to determine whether there is any risk +\fBTcl_UtfNext\fR might read beyond the readable memory occupied +by the string. .PP \fBTcl_UtfPrev\fR is used to step backward through but not beyond the UTF-8 string that begins at \fIstart\fR. If the UTF-8 string is made up entirely of complete and well-formed characters, and \fIsrc\fR points to the lead byte of one of those characters (or to the location one byte Index: generic/tclCmdMZ.c ================================================================== --- generic/tclCmdMZ.c +++ generic/tclCmdMZ.c @@ -2527,11 +2527,11 @@ if (!Tcl_UniCharIsWordChar(ch)) { break; } - next = Tcl_UtfPrev(p, string); + next = TclUtfPrev(p, string); do { next += delta; delta = TclUtfToUCS4(next, &ch); } while (next + delta < p); p = next; Index: generic/tclCompExpr.c ================================================================== --- generic/tclCompExpr.c +++ generic/tclCompExpr.c @@ -1891,11 +1891,10 @@ Tcl_Obj **literalPtr) /* Write corresponding literal value to this storage, if non-NULL. */ { const char *end; int scanned; - Tcl_UniChar ch = 0; Tcl_Obj *literal = NULL; unsigned char byte; if (numBytes == 0) { *lexemePtr = END; @@ -2099,17 +2098,17 @@ * have no direct relevance here. */ if (!TclIsBareword(*start) || *start == '_') { if (Tcl_UtfCharComplete(start, numBytes)) { - scanned = TclUtfToUniChar(start, &ch); + scanned = TclUtfNext(start) - start; } else { char utfBytes[4]; memcpy(utfBytes, start, numBytes); utfBytes[numBytes] = '\0'; - scanned = TclUtfToUniChar(utfBytes, &ch); + scanned = TclUtfNext(utfBytes) - utfBytes; } *lexemePtr = INVALID; Tcl_DecrRefCount(literal); return scanned; } Index: generic/tclInt.h ================================================================== --- generic/tclInt.h +++ generic/tclInt.h @@ -4687,10 +4687,18 @@ _count += Tcl_NumUtfChars((bytes) + _count, _i); \ } \ (numChars) = _count; \ } while (0); +#define TclUtfPrev(src, start) \ + (((src) < (start)+2) ? (start) : \ + ((unsigned char) *(src - 1)) < 0x80 ? (src)-1 : \ + Tcl_UtfPrev(src, start)) + +#define TclUtfNext(src) \ + ((((unsigned char) *(src)) < 0xC0) ? src + 1 : Tcl_UtfNext(src)) + /* *---------------------------------------------------------------- * Macro that encapsulates the logic that determines when it is safe to * interpret a string as a byte array directly. In summary, the object must be * a byte array and must not have a string representation (as the operations Index: generic/tclStringObj.c ================================================================== --- generic/tclStringObj.c +++ generic/tclStringObj.c @@ -1169,14 +1169,14 @@ if (ellipsis == NULL) { ellipsis = "..."; } eLen = strlen(ellipsis); while (eLen > limit) { - eLen = Tcl_UtfPrev(ellipsis+eLen, ellipsis) - ellipsis; + eLen = TclUtfPrev(ellipsis+eLen, ellipsis) - ellipsis; } - toCopy = Tcl_UtfPrev(bytes+limit+1-eLen, bytes) - bytes; + toCopy = TclUtfPrev(bytes+limit+1-eLen, bytes) - bytes; } /* * If objPtr has a valid Unicode rep, then append the Unicode conversion * of "bytes" to the objPtr's Unicode rep, otherwise append "bytes" to @@ -2612,11 +2612,11 @@ * Within that buffer, we trim both ends if needed so that we * copy only whole characters, and avoid copying any partial * multi-byte characters. */ - q = Tcl_UtfPrev(end, bytes); + q = TclUtfPrev(end, bytes); if (!Tcl_UtfCharComplete(q, (int)(end - q))) { end = q; } q = bytes + 4; Index: generic/tclUtf.c ================================================================== --- generic/tclUtf.c +++ generic/tclUtf.c @@ -64,11 +64,11 @@ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,1,1,1,1,1,1,1,1,1,1,1 }; static const unsigned char complete[256] = { 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, @@ -77,13 +77,14 @@ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* Tcl_UtfCharComplete() might point to 2nd byte of valid 4-byte sequence */ 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, /* End of "continuation byte section" */ - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, + 2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,1,1,1,1,1,1,1,1,1,1,1 }; +static int Overlong(unsigned char *src); /* *--------------------------------------------------------------------------- * * TclUtfCount -- @@ -112,11 +113,64 @@ if (((unsigned)(ch - 0x10000) <= 0xFFFFF)) { return 4; } return 3; } + +/* + *--------------------------------------------------------------------------- + * + * Overlong -- + * + * Utility routine to report whether /src/ points to the start of an + * overlong byte sequence that should be rejected. Caller guarantees + * that src[0] and src[1] are readable, and + * + * (src[0] >= 0xC0) && (src[0] != 0xC1) + * (src[1] >= 0x80) && (src[1] < 0xC0) + * (src[0] < ((TCL_UTF_MAX > 3) ? 0xF8 : 0xF0)) + * + * Results: + * A boolean. + *--------------------------------------------------------------------------- + */ +static CONST unsigned char overlong[3] = { + 0x80, /* \xD0 -- all sequences valid */ + 0xA0, /* \xE0\x80 through \xE0\x9F are invalid prefixes */ +#if TCL_UTF_MAX >= 3 + 0x90 /* \xF0\x80 through \xF0\x8F are invalid prefixes */ +#else + 0xC0 /* Not used, but reject all again for safety. */ +#endif +}; + +INLINE static int +Overlong( + unsigned char *src) /* Points to lead byte of a UTF-8 byte sequence */ +{ + unsigned char byte = *src; + + if (byte % 0x10) { + /* Only lead bytes 0xC0, 0xE0, 0xF0 need examination */ + return 0; + } + if (byte == 0xC0) { + if (src[1] == 0x80) { + /* Valid sequence: \xC0\x80 for \u0000 */ + return 0; + } + /* Reject overlong: \xC0\x81 - \xC0\xBF */ + return 1; + } + if (src[1] < overlong[(byte >> 4) - 0x0D]) { + /* Reject overlong */ + return 1; + } + return 0; +} + /* *--------------------------------------------------------------------------- * * Tcl_UniCharToUtf -- * @@ -733,11 +787,11 @@ Tcl_NumUtfChars( const char *src, /* The UTF-8 string to measure. */ int length) /* The length of the string in bytes, or -1 * for strlen(string). */ { - Tcl_UniChar ch = 0; + const char *next; int i = 0; /* * The separate implementations are faster. * @@ -744,26 +798,28 @@ * Since this is a time-sensitive function, we also do the check for the * single-byte char case specially. */ if (length < 0) { - while (*src != '\0') { - src += TclUtfToUniChar(src, &ch); - i++; + while ((*src != '\0') && (i < INT_MAX)) { + next = TclUtfNext(src); + i += 1 + ((next - src) > 3); + src = next; } - if (i < 0) i = INT_MAX; /* Bug [2738427] */ } else { const char *endPtr = src + length - 4; while (src < endPtr) { - src += TclUtfToUniChar(src, &ch); - i++; + next = TclUtfNext(src); + i += 1 + ((next - src) > 3); + src = next; } endPtr += 4; while ((src < endPtr) && Tcl_UtfCharComplete(src, endPtr - src)) { - src += TclUtfToUniChar(src, &ch); - i++; + next = TclUtfNext(src); + i += 1 + ((next - src) > 3); + src = next; } if (src < endPtr) { i += endPtr - src; } } @@ -850,17 +906,39 @@ /* *--------------------------------------------------------------------------- * * Tcl_UtfNext -- * - * Given a pointer to some current location in a UTF-8 string, move - * forward one character. The caller must ensure that they are not asking - * for the next character after the last character in the string. + * The aim of this routine is to provide a way to iterate forward + * through a UTF-8 string. The caller is expected to pass a non-NULL + * pointer argument /src/ which points to a location within a string. + * (*src) will be read, so /src/ must not point to an unreadable + * location past the end of the string. If /src/ points to the + * beginning of a complete, well-formed and valid UTF_8 byte sequence + * of no more than TCL_UTF_MAX bytes, Tcl_UtfNext returns the pointer + * just past the end of that sequence. In any other circumstance, + * Tcl_UtfNext returns /src/+1. + * + * Because this routine always returns a value > /src/, it is useful + * as a forward iterator that will always make progress. If the string + * is NUL-terminated, Tcl_UtfNext will not read beyond the terminating + * NUL character. If it is not NUL-terminated, the caller must make + * use of the companion routine Tcl_UtfCharComplete to test whether + * there is risk that Tcl_UtfNext will read beyond the end of the string. + * Tcl_UtfNext will never read more than TCL_UTF_MAX bytes. + * + * In a string where all characters are complete and properly formed, + * and /src/ points to the first byte of a character, repeated + * Tcl_UtfNext calls will step to the starting bytes of characters, one + * character at a time. Within those limitations, Tcl_UtfPrev and + * Tcl_UtfNext are inverses. If either condition cannot be met, + * Tcl_UtfPrev and Tcl_UtfNext may not function as inverses and the + * caller will have to take greater care. * * Results: - * The return value is the pointer to the next character in the UTF-8 - * string. + * A pointer to the start of the next character in the string (or to + * the end of the string) as described above. * * Side effects: * None. * *--------------------------------------------------------------------------- @@ -868,59 +946,70 @@ const char * Tcl_UtfNext( const char *src) /* The current location in the string. */ { - Tcl_UniChar ch = 0; - int len = TclUtfToUniChar(src, &ch); - -#if TCL_UTF_MAX <= 3 - if ((ch >= 0xD800) && (len < 3)) { - len += TclUtfToUniChar(src + len, &ch); - } -#endif - return src + len; + int byte = *((unsigned char *) src); + int left = totalBytes[byte]; + const char *next = src + 1; + + while (--left) { + byte = *((unsigned char *) next); + if ((byte & 0xC0) != 0x80) { + /* + * src points to non-trail byte; We ran out of trail bytes + * before the needs of the lead byte were satisfied. + * Let the (malformed) lead byte alone be a character + */ + return src + 1; + } + next++; + } + if (Overlong((unsigned char *)src)) { + return src + 1; + } + return next; } /* *--------------------------------------------------------------------------- * * Tcl_UtfPrev -- * * The aim of this routine is to provide a way to move backward * through a UTF-8 string. The caller is expected to pass non-NULL - * pointer arguments start and src. start points to the beginning - * of a string, and src >= start points to a location within (or just - * past the end) of the string. This routine always returns a - * pointer within the string (>= start). When (src == start), it - * returns start. When (src > start), it returns a pointer (< src) - * and (>= src - TCL_UTF_MAX). Subject to these constraints, the - * routine returns a pointer to the earliest byte in the string that - * starts a character when characters are read starting at start and + * pointer arguments /start/ and /src/. /start/ points to the beginning + * of a string, and /src/ (>= /start/) points to a location within (or + * just past the end) of the string. This routine always returns a + * pointer within the string (>= /start/). When (/src/ == /start/), + * it returns /start/. When (/src/ > /start/), it returns a pointer + * (< /src/) and (>= /src/ - TCL_UTF_MAX). Subject to these constraints, + * the routine returns a pointer to the earliest byte in the string that + * starts a character when characters are read starting at /start/ and * that character might include the byte src[-1]. The routine will * examine only those bytes in the range that might be returned. - * It will not examine the byte *src, and because of that cannot + * It will not examine the byte (*src), and because of that cannot * determine for certain in all circumstances whether the character * that begins with the returned pointer will or will not include - * the byte src[-1]. In the scenario, where src points to the end of - * a buffer being filled, the returned pointer point to either the + * the byte src[-1]. In the scenario where /src/ points to the end of + * a buffer being filled, the returned pointer points to either the * final complete character in the string or to the earliest byte * that might start an incomplete character waiting for more bytes to * complete. * - * Because this routine always returns a value < src until the point - * it is forced to return start, it is useful as a backward iterator + * Because this routine always returns a value < /src/ until the point + * it is forced to return /start/, it is useful as a backward iterator * through a string that will always make progress and always be * prevented from running past the beginning of the string. * * In a string where all characters are complete and properly formed, - * and the value of src points to the first byte of a character, - * repeated Tcl_UtfPrev calls will step to the starting bytes of - * characters, one character at a time. Within those limitations, - * Tcl_UtfPrev and Tcl_UtfNext are inverses. If either condition cannot - * be met, Tcl_UtfPrev and Tcl_UtfNext may not function as inverses and - * the caller will have to take greater care. + * and /src/ points to the first byte of a character, repeated + * Tcl_UtfPrev calls will step to the starting bytes of characters, one + * character at a time. Within those limitations, Tcl_UtfPrev and + * Tcl_UtfNext are inverses. If either condition cannot be met, + * Tcl_UtfPrev and Tcl_UtfNext may not function as inverses and the + * caller will have to take greater care. * * Results: * A pointer to the start of a character in the string as described * above. * @@ -933,34 +1022,96 @@ const char * Tcl_UtfPrev( const char *src, /* A location in a UTF-8 string. */ const char *start) /* Pointer to the beginning of the string */ { - const char *look; - int i, byte; - - look = --src; - for (i = 0; i < 4; i++) { - if (look < start) { - if (src < start) { - src = start; - } - break; - } - byte = *((unsigned char *) look); + int trailBytesSeen = 0; /* How many trail bytes have been verified? */ + CONST char *fallback = src - 1; + /* If we cannot find a lead byte that might + * start a prefix of a valid UTF byte sequence, + * we will fallback to a one-byte back step */ + unsigned char *look = (unsigned char *)fallback; + /* Start search at the fallback position */ + + /* Quick boundary case exit. */ + if (fallback <= start) { + return start; + } + + do { + unsigned char byte = look[0]; + if (byte < 0x80) { - break; + /* + * Single byte character. Either this is a correct previous + * character, or it is followed by at least one trail byte + * which indicates a malformed sequence. In either case the + * correct result is to return the fallback. + */ + return fallback; } if (byte >= 0xC0) { - if (totalBytes[byte] <= i) { - break; + /* Non-trail byte; May be multibyte lead. */ + + if ((trailBytesSeen == 0) + /* + * We've seen no trailing context to use to check + * anything. From what we know, this non-trail byte + * is a prefix of a previous character, and accepting + * it (the fallback) is correct. + */ + + || (trailBytesSeen >= totalBytes[byte])) { + /* + * That is, (1 + trailBytesSeen > needed). + * We've examined more bytes than needed to complete + * this lead byte. No matter about well-formedness or + * validity, the sequence starting with this lead byte + * will never include the fallback location, so we must + * return the fallback location. See test utf-7.17 + */ + return fallback; + } + + /* + * trailBytesSeen > 0, so we can examine look[1] safely. + * Use that capability to screen out overlong sequences. + */ + + if (Overlong(look)) { + /* Reject */ + return fallback; } - return look; + return (CONST char *)look; + } + + /* We saw a trail byte. */ + trailBytesSeen++; + + if ((CONST char *)look == start) { + /* + * Do not read before the start of the string + * + * If we get here, we've examined bytes at every location + * >= start and < src and all of them are trail bytes, + * including (*start). We need to return our fallback + * and exit this loop before we run past the start of the string. + */ + return fallback; } + + /* Continue the search backwards... */ look--; - } - return src; + } while (trailBytesSeen < /* was TCL_UTF_MAX */ 4); + + /* + * We've seen 4 (was TCL_UTF_MAX) trail bytes, so we know there will not be a + * properly formed byte sequence to find, and we can stop looking, + * accepting the fallback. + */ + + return fallback; } /* *--------------------------------------------------------------------------- * @@ -981,32 +1132,14 @@ int Tcl_UniCharAtIndex( const char *src, /* The UTF-8 string to dereference. */ int index) /* The position of the desired character. */ { - Tcl_UniChar ch = 0; - int fullchar = 0; -#if TCL_UTF_MAX <= 3 - int len = 0; -#endif - - while (index-- >= 0) { -#if TCL_UTF_MAX <= 3 - src += (len = TclUtfToUniChar(src, &ch)); -#else - src += TclUtfToUniChar(src, &ch); -#endif - } - fullchar = ch; -#if TCL_UTF_MAX <= 3 - if ((ch >= 0xD800) && (len < 3)) { - /* If last Tcl_UniChar was a high surrogate, combine with low surrogate */ - (void)TclUtfToUniChar(src, &ch); - fullchar = (((fullchar & 0x3FF) << 10) | (ch & 0x3FF)) + 0x10000; - } -#endif - return fullchar; + int ch; + + TclUtfToUCS4(Tcl_UtfAtIndex(src, index), &ch); + return ch; } /* *--------------------------------------------------------------------------- * @@ -1029,23 +1162,22 @@ const char * Tcl_UtfAtIndex( const char *src, /* The UTF-8 string. */ int index) /* The position of the desired character. */ { - Tcl_UniChar ch = 0; - int len = 0; - while (index-- > 0) { - len = TclUtfToUniChar(src, &ch); - src += len; - } -#if TCL_UTF_MAX <= 3 - if ((ch >= 0xD800) && (len < 3)) { - /* Index points at character following high Surrogate */ - src += TclUtfToUniChar(src, &ch); - } -#endif + const char *next = TclUtfNext(src); + + /* + * 4-byte sequences generate two UCS-2 code units in the + * UTF-16 representation, so in the current indexing scheme + * we need to account for an extra index (total of two). + */ + index -= ((next - src) > 3); + + src = next; + } return src; } /* *--------------------------------------------------------------------------- Index: generic/tclUtil.c ================================================================== --- generic/tclUtil.c +++ generic/tclUtil.c @@ -1705,11 +1705,11 @@ do { const char *q = trim; int pInc = 0, bytesLeft = numTrim; - pp = Tcl_UtfPrev(p, bytes); + pp = TclUtfPrev(p, bytes); do { pp += pInc; pInc = TclUtfToUCS4(pp, &ch1); } while (pp + pInc < p); @@ -1856,11 +1856,11 @@ /* If we did not trim the whole string, it starts with a character * that we will not trim. Skip over it. */ if (numBytes > 0) { const char *first = bytes + trimLeft; - bytes = Tcl_UtfNext(first); + bytes = TclUtfNext(first); numBytes -= (bytes - first); if (numBytes > 0) { /* When bytes is NUL-terminated, returns * 0 <= trimRight <= numBytes */ Index: generic/tclZlib.c ================================================================== --- generic/tclZlib.c +++ generic/tclZlib.c @@ -3074,11 +3074,13 @@ * The zlib transform allows us to read at most one character from the * underlying channel to properly identify Z_STREAM_END without * reading over the border. */ - readBytes = Tcl_ReadRaw(cd->parent, cd->inBuffer, cd->readAheadLimit); + readBytes = Tcl_ReadRaw(cd->parent, cd->inBuffer, + cd->readAheadLimit <= cd->inAllocated ? + cd->readAheadLimit : cd->inAllocated); /* * Three cases here: * 1. Got some data from the underlying channel (readBytes > 0) so * it should be fed through the decompression engine. Index: tests/utf.test ================================================================== --- tests/utf.test +++ tests/utf.test @@ -93,11 +93,11 @@ test utf-2.10 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail, underflow} testbytestring { string length [testbytestring "\xF0\x8F\xBF\xBF"] } {4} test utf-2.11 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail, overflow} testbytestring { string length [testbytestring "\xF4\x90\x80\x80"] -} {4} +} {2} test utf-2.12 {Tcl_UtfToUniChar: longer UTF sequences not supported} testbytestring { string length [testbytestring "\xF8\xA2\xA2\xA2\xA2"] } {5} test utf-3.1 {Tcl_UtfCharComplete} { @@ -150,12 +150,278 @@ } {bcbc} test utf-5.2 {Tcl_UtfFindLast} {testfindlast testbytestring} { testfindlast [testbytestring "abcbc"] 98 } {bc} -test utf-6.1 {Tcl_UtfNext} { -} {} +testConstraint testutfnext [llength [info commands testutfnext]] + +test utf-6.1 {Tcl_UtfNext} testutfnext { + # This takes the pointer one past the terminating NUL. + # This is really an invalid call. + testutfnext {} +} 1 +test utf-6.2 {Tcl_UtfNext} testutfnext { + testutfnext A +} 1 +test utf-6.3 {Tcl_UtfNext} testutfnext { + testutfnext AA +} 1 +test utf-6.4 {Tcl_UtfNext} testutfnext { + testutfnext A\xA0 +} 1 +test utf-6.5 {Tcl_UtfNext} testutfnext { + testutfnext A\xD0 +} 1 +test utf-6.6 {Tcl_UtfNext} testutfnext { + testutfnext A\xE8 +} 1 +test utf-6.7 {Tcl_UtfNext} testutfnext { + testutfnext A\xF4 +} 1 +test utf-6.8 {Tcl_UtfNext} testutfnext { + testutfnext A\xF8 +} 1 +test utf-6.9 {Tcl_UtfNext} testutfnext { + testutfnext \xA0 +} 1 +test utf-6.10 {Tcl_UtfNext} testutfnext { + testutfnext \xA0G +} 1 +test utf-6.11 {Tcl_UtfNext} testutfnext { + testutfnext \xA0\xA0 +} 1 +test utf-6.12 {Tcl_UtfNext} testutfnext { + testutfnext \xA0\xD0 +} 1 +test utf-6.13 {Tcl_UtfNext} testutfnext { + testutfnext \xA0\xE8 +} 1 +test utf-6.14 {Tcl_UtfNext} testutfnext { + testutfnext \xA0\xF4 +} 1 +test utf-6.15 {Tcl_UtfNext} testutfnext { + testutfnext \xA0\xF8 +} 1 +test utf-6.16 {Tcl_UtfNext} testutfnext { + testutfnext \xD0 +} 1 +test utf-6.17 {Tcl_UtfNext} testutfnext { + testutfnext \xD0A +} 1 +test utf-6.18 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xA0 +} 2 +test utf-6.19 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xD0 +} 1 +test utf-6.20 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xE8 +} 1 +test utf-6.21 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xF4 +} 1 +test utf-6.22 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xF8 +} 1 +test utf-6.23 {Tcl_UtfNext} testutfnext { + testutfnext \xE8 +} 1 +test utf-6.24 {Tcl_UtfNext} testutfnext { + testutfnext \xE8A +} 1 +test utf-6.25 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0 +} 1 +test utf-6.26 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xD0 +} 1 +test utf-6.27 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xE8 +} 1 +test utf-6.28 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xF4 +} 1 +test utf-6.29 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xF8 +} 1 +test utf-6.30 {Tcl_UtfNext} testutfnext { + testutfnext \xF4 +} 1 +test utf-6.31 {Tcl_UtfNext} testutfnext { + testutfnext \xF4A +} 1 +test utf-6.32 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0 +} 1 +test utf-6.33 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xD0 +} 1 +test utf-6.34 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xE8 +} 1 +test utf-6.35 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xF4 +} 1 +test utf-6.36 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xF8 +} 1 +test utf-6.37 {Tcl_UtfNext} testutfnext { + testutfnext \xF8 +} 1 +test utf-6.38 {Tcl_UtfNext} testutfnext { + testutfnext \xF8A +} 1 +test utf-6.39 {Tcl_UtfNext} testutfnext { + testutfnext \xF8\xA0 +} 1 +test utf-6.40 {Tcl_UtfNext} testutfnext { + testutfnext \xF8\xD0 +} 1 +test utf-6.41 {Tcl_UtfNext} testutfnext { + testutfnext \xF8\xE8 +} 1 +test utf-6.42 {Tcl_UtfNext} testutfnext { + testutfnext \xF8\xF4 +} 1 +test utf-6.43 {Tcl_UtfNext} testutfnext { + testutfnext \xF8\xF8 +} 1 +test utf-6.44 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xA0G +} 2 +test utf-6.45 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xA0\xA0 +} 2 +test utf-6.46 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xA0\xD0 +} 2 +test utf-6.47 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xA0\xE8 +} 2 +test utf-6.48 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xA0\xF4 +} 2 +test utf-6.49 {Tcl_UtfNext} testutfnext { + testutfnext \xD0\xA0\xF8 +} 2 +test utf-6.50 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0G +} 1 +test utf-6.51 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xA0 +} 3 +test utf-6.52 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xD0 +} 1 +test utf-6.53 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xE8 +} 1 +test utf-6.54 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xF4 +} 1 +test utf-6.55 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xF8 +} 1 +test utf-6.56 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0G +} 1 +test utf-6.57 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0 +} 1 +test utf-6.58 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xD0 +} 1 +test utf-6.59 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xE8 +} 1 +test utf-6.60 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xF4 +} 1 +test utf-6.61 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xF8 +} 1 +test utf-6.62 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xA0G +} 3 +test utf-6.63 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xA0\xA0 +} 3 +test utf-6.64 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xA0\xD0 +} 3 +test utf-6.65 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xA0\xE8 +} 3 +test utf-6.66 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xA0\xF4 +} 3 +test utf-6.67 {Tcl_UtfNext} testutfnext { + testutfnext \xE8\xA0\xA0\xF8 +} 3 +test utf-6.68 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0G +} 1 +test utf-6.69 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xA0 +} 4 +test utf-6.70 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xD0 +} 1 +test utf-6.71 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xE8 +} 1 +test utf-6.71 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xF4 +} 1 +test utf-6.73 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xF8 +} 1 +test utf-6.74 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xA0G +} 4 +test utf-6.75 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xA0\xA0 +} 4 +test utf-6.76 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xA0\xD0 +} 4 +test utf-6.77 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xA0\xE8 +} 4 +test utf-6.78 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xA0\xF4 +} 4 +test utf-6.79 {Tcl_UtfNext} testutfnext { + testutfnext \xF4\xA0\xA0\xA0G\xF8 +} 4 +test utf-6.80 {Tcl_UtfNext - overlong sequences} testutfnext { + testutfnext \xC0\x80 +} 2 +test utf-6.81 {Tcl_UtfNext - overlong sequences} testutfnext { + testutfnext \xC0\x81 +} 1 +test utf-6.82 {Tcl_UtfNext - overlong sequences} testutfnext { + testutfnext \xC1\x80 +} 1 +test utf-6.83 {Tcl_UtfNext - overlong sequences} testutfnext { + testutfnext \xC2\x80 +} 2 +test utf-6.84 {Tcl_UtfNext - overlong sequences} testutfnext { + testutfnext \xE0\x80\x80 +} 1 +test utf-6.85 {Tcl_UtfNext - overlong sequences} testutfnext { + testutfnext \xE0\xA0\x80 +} 3 +test utf-6.86 {Tcl_UtfNext - overlong sequences} testutfnext { + testutfnext \xF0\x80\x80\x80 +} 1 +test utf-6.87 {Tcl_UtfNext - overlong sequences} {testutfnext} { + testutfnext \xF0\x90\x80\x80 +} 4 +test utf-6.88 {Tcl_UtfNext, pointing to 2th byte of 3-byte valid sequence} {testutfnext} { + testutfnext \xE8\xA0\xA0 1 +} 2 testConstraint testutfprev [llength [info commands testutfprev]] test utf-7.1 {Tcl_UtfPrev} testutfprev { testutfprev {} @@ -314,10 +580,79 @@ testutfprev A\xD0\xA0\xA0\xA0 } 4 test utf-7.23 {Tcl_UtfPrev} testutfprev { testutfprev A\xA0\xA0\xA0\xA0 } 4 +test utf-7.24 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xC0\x81 +} 2 +test utf-7.25 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xC0\x81 2 +} 1 +test utf-7.26 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xE0\x80\x80 +} 3 +test utf-7.27 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xE0\x80\x80 3 +} 2 +test utf-7.28 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xE0\x80\x80 2 +} 1 +test utf-7.29 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xF0\x80\x80\x80 +} 4 +test utf-7.30 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xF0\x80\x80\x80 4 +} 3 +test utf-7.31 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xF0\x80\x80\x80 3 +} 2 +test utf-7.32 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xF0\x80\x80\x80 2 +} 1 +test utf-7.33 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xC0\x80 +} 1 +test utf-7.34 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xC1\x80 +} 2 +test utf-7.35 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xC2\x80 +} 1 +test utf-7.36 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xE0\xA0\x80 +} 1 +test utf-7.37 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xE0\xA0\x80 3 +} 1 +test utf-7.38 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xE0\xA0\x80 2 +} 1 +test utf-7.39 {Tcl_UtfPrev -- overlong sequence} {testutfprev} { + testutfprev A\xF0\x90\x80\x80 +} 1 +test utf-7.40 {Tcl_UtfPrev -- overlong sequence} {testutfprev} { + testutfprev A\xF0\x90\x80\x80 4 +} 1 +test utf-7.41 {Tcl_UtfPrev -- overlong sequence} {testutfprev} { + testutfprev A\xF0\x90\x80\x80 3 +} 1 +test utf-7.42 {Tcl_UtfPrev -- overlong sequence} testutfprev { + testutfprev A\xF0\x90\x80\x80 2 +} 1 +test utf-7.43 {Tcl_UtfPrev -- no lead byte at start} testutfprev { + testutfprev \xA0 +} 0 +test utf-7.44 {Tcl_UtfPrev -- no lead byte at start} testutfprev { + testutfprev \xA0\xA0 +} 1 +test utf-7.45 {Tcl_UtfPrev -- no lead byte at start} testutfprev { + testutfprev \xA0\xA0\xA0 +} 2 +test utf-7.46 {Tcl_UtfPrev -- no lead byte at start} testutfprev { + testutfprev \xA0\xA0\xA0\xA0 +} 3 test utf-8.1 {Tcl_UniCharAtIndex: index = 0} { string index abcd 0 } {a} test utf-8.2 {Tcl_UniCharAtIndex: index = 0} {