Diff
Not logged in

Differences From Artifact [9457d8b6d7]:

To Artifact [ee720bf978]:


92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
};

/*
 * Functions used only in this module.
 */

static int		Invalid(unsigned char *src);

#define UCS4ToUpper Tcl_UniCharToUpper
#define UCS4ToLower Tcl_UniCharToLower
#define UCS4ToTitle Tcl_UniCharToTitle


/*
 *---------------------------------------------------------------------------
 *
 * TclUtfCount --
 *
 *	Find the number of bytes in the Utf character "ch".







<
<
<
<
<







92
93
94
95
96
97
98





99
100
101
102
103
104
105
};

/*
 * Functions used only in this module.
 */

static int		Invalid(unsigned char *src);






/*
 *---------------------------------------------------------------------------
 *
 * TclUtfCount --
 *
 *	Find the number of bytes in the Utf character "ch".
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
static int
Invalid(
    unsigned char *src)	/* Points to lead byte of a UTF-8 byte sequence */
{
    unsigned char byte = *src;
    int index;

    if (byte % 0x04) {
	/* Only lead bytes 0xC0, 0xE0, 0xF0, 0xF4 need examination */
	return 0;
    }
    index = (byte - 0xC0) >> 1;
    if (src[1] < bounds[index] || src[1] > bounds[index+1]) {
	/* Out of bounds - report invalid. */
	return 1;







|







162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
static int
Invalid(
    unsigned char *src)	/* Points to lead byte of a UTF-8 byte sequence */
{
    unsigned char byte = *src;
    int index;

    if ((byte & 0xC3) != 0xC0) {
	/* Only lead bytes 0xC0, 0xE0, 0xF0, 0xF4 need examination */
	return 0;
    }
    index = (byte - 0xC0) >> 1;
    if (src[1] < bounds[index] || src[1] > bounds[index+1]) {
	/* Out of bounds - report invalid. */
	return 1;
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
   0x2DC, 0x2122, 0x0161, 0x203A, 0x0153,   0x9D, 0x017E, 0x0178
};

#undef Tcl_UtfToUniChar
int
Tcl_UtfToUniChar(
    const char *src,	/* The UTF-8 string. */
    int *chPtr)/* Filled with the unsigned int represented by
				 * the UTF-8 string. */
{
    int byte;

    /*
     * Unroll 1 to 4 byte UTF-8 sequences.
     */







|







434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
   0x2DC, 0x2122, 0x0161, 0x203A, 0x0153,   0x9D, 0x017E, 0x0178
};

#undef Tcl_UtfToUniChar
int
Tcl_UtfToUniChar(
    const char *src,	/* The UTF-8 string. */
    int *chPtr)/* Filled with the Unicode character represented by
				 * the UTF-8 string. */
{
    int byte;

    /*
     * Unroll 1 to 4 byte UTF-8 sequences.
     */
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
	 */
    }
    else if (byte < 0xF8) {
	if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80) && ((src[3] & 0xC0) == 0x80)) {
	    /*
	     * Four-byte-character lead byte followed by three trail bytes.
	     */
	    *chPtr = (((byte & 0x07) << 18) | ((src[1] & 0x3F) << 12)
		    | ((src[2] & 0x3F) << 6) | (src[3] & 0x3F));
	    if ((unsigned)(*chPtr - 0x10000) <= 0xFFFFF) {







|







496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
	 */
    }
    else if (byte < 0xF5) {
	if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80) && ((src[3] & 0xC0) == 0x80)) {
	    /*
	     * Four-byte-character lead byte followed by three trail bytes.
	     */
	    *chPtr = (((byte & 0x07) << 18) | ((src[1] & 0x3F) << 12)
		    | ((src[2] & 0x3F) << 6) | (src[3] & 0x3F));
	    if ((unsigned)(*chPtr - 0x10000) <= 0xFFFFF) {
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
    *chPtr = byte;
    return 1;
}

int
Tcl_UtfToChar16(
    const char *src,	/* The UTF-8 string. */
    unsigned short *chPtr)/* Filled with the unsigned short represented by
				 * the UTF-8 string. */
{
    unsigned short byte;

    /*
     * Unroll 1 to 4 byte UTF-8 sequences.
     */








|
|







521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
    *chPtr = byte;
    return 1;
}

int
Tcl_UtfToChar16(
    const char *src,	/* The UTF-8 string. */
    unsigned short *chPtr)/* Filled with the Tcl_UniChar represented by
				 * the UTF-8 string. This could be a surrogate too. */
{
    unsigned short byte;

    /*
     * Unroll 1 to 4 byte UTF-8 sequences.
     */

600
601
602
603
604
605
606
607
608
609
610

611
612
613
614
615
616
617
618
619
620

621
622
623
624
625
626
627
	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
	 */
    }
    else if (byte < 0xF8) {
	if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80) && ((src[3] & 0xC0) == 0x80)) {
	    /*
	     * Four-byte-character lead byte followed by three trail bytes.

	     */
	    unsigned short high = (((byte & 0x07) << 8) | ((src[1] & 0x3F) << 2)
		    | ((src[2] & 0x3F) >> 4)) - 0x40;
	    if (high >= 0x400) {
		/* out of range, < 0x10000 or > 0x10FFFF */
	    } else {
		/* produce high surrogate, advance source pointer */
		*chPtr = 0xD800 + high;
		return 1;
	    }

	}

	/*
	 * A four-byte-character lead-byte not followed by three trail-bytes
	 * represents itself.
	 */
    }







|
|

|
>

|

|
<
<




>







595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610


611
612
613
614
615
616
617
618
619
620
621
622
	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
	 */
    }
    else if (byte < 0xF5) {
	if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80)) {
	    /*
	     * Four-byte-character lead byte followed by at least two trail bytes.
	     * We don't test the validity of 3th trail byte, see [ed29806ba]
	     */
	    Tcl_UniChar high = (((byte & 0x07) << 8) | ((src[1] & 0x3F) << 2)
		    | ((src[2] & 0x3F) >> 4)) - 0x40;
	    if (high < 0x400) {


		/* produce high surrogate, advance source pointer */
		*chPtr = 0xD800 + high;
		return 1;
	    }
	    /* out of range, < 0x10000 or > 0x10FFFF */
	}

	/*
	 * A four-byte-character lead-byte not followed by three trail-bytes
	 * represents itself.
	 */
    }
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792

int
Tcl_UtfCharComplete(
    const char *src,		/* String to check if first few bytes contain
				 * a complete UTF-8 character. */
    int length)			/* Length of above string in bytes. */
{
    return length >= complete[(unsigned char)*src];
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_NumUtfChars --
 *







|







773
774
775
776
777
778
779
780
781
782
783
784
785
786
787

int
Tcl_UtfCharComplete(
    const char *src,		/* String to check if first few bytes contain
				 * a complete UTF-8 character. */
    int length)			/* Length of above string in bytes. */
{
    return length >= complete[UCHAR(*src)];
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_NumUtfChars --
 *
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
    if (length < 0) {
	while (*src != '\0') {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	if (i < 0) i = INT_MAX; /* Bug [2738427] */
    } else {
	const char *endPtr = src + length - 4;

	while (src < endPtr) {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	endPtr += 4;
	while ((src < endPtr) && Tcl_UtfCharComplete(src, endPtr - src)) {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	if (src < endPtr) {
	    i += endPtr - src;
	}







|





|







817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
    if (length < 0) {
	while (*src != '\0') {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	if (i < 0) i = INT_MAX; /* Bug [2738427] */
    } else {
	const char *endPtr = src + length - TCL_UTF_MAX;

	while (src < endPtr) {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	endPtr += TCL_UTF_MAX;
	while ((src < endPtr) && Tcl_UtfCharComplete(src, endPtr - src)) {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	if (src < endPtr) {
	    i += endPtr - src;
	}
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
	     * before the needs of the lead byte were satisfied.
	     * Let the (malformed) lead byte alone be a character
	     */
	    return src + 1;
	}
	next++;
    }
    if ((next == src + 1) || Invalid((unsigned char *)src)) {
	return src + 1;
    }
    return next;
}

/*
 *---------------------------------------------------------------------------







|







958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
	     * before the needs of the lead byte were satisfied.
	     * Let the (malformed) lead byte alone be a character
	     */
	    return src + 1;
	}
	next++;
    }
    if (Invalid((unsigned char *)src)) {
	return src + 1;
    }
    return next;
}

/*
 *---------------------------------------------------------------------------
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086

1087



1088
1089
1090
1091
1092
1093
1094
	     * and exit this loop before we run past the start of the string.
	     */
	    return fallback;
	}

	/* Continue the search backwards... */
	look--;
    } while (trailBytesSeen < 4);

    /*
     * We've seen TCL_UTF_MAX trail bytes, so we know there will not be a
     * properly formed byte sequence to find, and we can stop looking,
     * accepting the fallback (for TCL_UTF_MAX > 3) or just go back as
     * far as we can.
     */

    return fallback;



}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_UniCharAtIndex --
 *







|







>

>
>
>







1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
	     * and exit this loop before we run past the start of the string.
	     */
	    return fallback;
	}

	/* Continue the search backwards... */
	look--;
    } while (trailBytesSeen < TCL_UTF_MAX);

    /*
     * We've seen TCL_UTF_MAX trail bytes, so we know there will not be a
     * properly formed byte sequence to find, and we can stop looking,
     * accepting the fallback (for TCL_UTF_MAX > 3) or just go back as
     * far as we can.
     */
#if TCL_UTF_MAX > 3
    return fallback;
#else
    return src - TCL_UTF_MAX;
#endif
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_UniCharAtIndex --
 *
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
    /*
     * Iterate over the string until we hit the terminating null.
     */

    src = dst = str;
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	upChar = UCS4ToUpper(ch);

	/*
	 * To keep badly formed Utf strings from getting inflated by the
	 * conversion (thereby causing a segfault), only copy the upper case
	 * char to dst if its size is <= the original char.
	 */

	if ((len < TclUtfCount(upChar)) || ((upChar & 0xF800) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(upChar, dst);
	}
	src += len;
    }







|







|







1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
    /*
     * Iterate over the string until we hit the terminating null.
     */

    src = dst = str;
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	upChar = Tcl_UniCharToUpper(ch);

	/*
	 * To keep badly formed Utf strings from getting inflated by the
	 * conversion (thereby causing a segfault), only copy the upper case
	 * char to dst if its size is <= the original char.
	 */

	if ((len < TclUtfCount(upChar)) || ((upChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(upChar, dst);
	}
	src += len;
    }
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
    /*
     * Iterate over the string until we hit the terminating null.
     */

    src = dst = str;
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	lowChar = UCS4ToLower(ch);

	/*
	 * To keep badly formed Utf strings from getting inflated by the
	 * conversion (thereby causing a segfault), only copy the lower case
	 * char to dst if its size is <= the original char.
	 */

	if ((len < TclUtfCount(lowChar)) || ((lowChar & 0xF800) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(lowChar, dst);
	}
	src += len;
    }







|







|







1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
    /*
     * Iterate over the string until we hit the terminating null.
     */

    src = dst = str;
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	lowChar = Tcl_UniCharToLower(ch);

	/*
	 * To keep badly formed Utf strings from getting inflated by the
	 * conversion (thereby causing a segfault), only copy the lower case
	 * char to dst if its size is <= the original char.
	 */

	if ((len < TclUtfCount(lowChar)) || ((lowChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(lowChar, dst);
	}
	src += len;
    }
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
     * characters until we get to a null.
     */

    src = dst = str;

    if (*src) {
	len = TclUtfToUCS4(src, &ch);
	titleChar = UCS4ToTitle(ch);

	if ((len < TclUtfCount(titleChar)) || ((titleChar & 0xF800) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(titleChar, dst);
	}
	src += len;
    }
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	lowChar = ch;
	/* Special exception for Georgian Asomtavruli chars, no titlecase. */
	if ((unsigned)(lowChar - 0x1C90) >= 0x30) {
	    lowChar = UCS4ToLower(lowChar);
	}

	if ((len < TclUtfCount(lowChar)) || ((lowChar & 0xF800) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(lowChar, dst);
	}
	src += len;
    }







|

|












|


|







1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
     * characters until we get to a null.
     */

    src = dst = str;

    if (*src) {
	len = TclUtfToUCS4(src, &ch);
	titleChar = Tcl_UniCharToTitle(ch);

	if ((len < TclUtfCount(titleChar)) || ((titleChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(titleChar, dst);
	}
	src += len;
    }
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	lowChar = ch;
	/* Special exception for Georgian Asomtavruli chars, no titlecase. */
	if ((unsigned)(lowChar - 0x1C90) >= 0x30) {
	    lowChar = Tcl_UniCharToLower(lowChar);
	}

	if ((len < TclUtfCount(lowChar)) || ((lowChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(lowChar, dst);
	}
	src += len;
    }
1658
1659
1660
1661
1662
1663
1664

1665
1666
1667
1668
1669
1670
1671
    if (!UNICODE_OUT_OF_RANGE(ch)) {
	int info = GetUniCharInfo(ch);

	if (GetCaseType(info) & 0x04) {
	    ch -= GetDelta(info);
	}
    }

    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharToLower --







>







1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
    if (!UNICODE_OUT_OF_RANGE(ch)) {
	int info = GetUniCharInfo(ch);

	if (GetCaseType(info) & 0x04) {
	    ch -= GetDelta(info);
	}
    }
    /* Clear away extension bits, if any */
    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharToLower --
1689
1690
1691
1692
1693
1694
1695

1696
1697
1698
1699
1700
1701
1702
	int info = GetUniCharInfo(ch);
	int mode = GetCaseType(info);

	if ((mode & 0x02) && (mode != 0x7)) {
	    ch += GetDelta(info);
	}
    }

    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharToTitle --







>







1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
	int info = GetUniCharInfo(ch);
	int mode = GetCaseType(info);

	if ((mode & 0x02) && (mode != 0x7)) {
	    ch += GetDelta(info);
	}
    }
    /* Clear away extension bits, if any */
    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharToTitle --
1728
1729
1730
1731
1732
1733
1734

1735
1736
1737
1738
1739
1740
1741
	    if (mode != 0x7) {
		ch += ((mode & 0x4) ? -1 : 1);
	    }
	} else if (mode == 0x4) {
	    ch -= GetDelta(info);
	}
    }

    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharLen --







>







1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
	    if (mode != 0x7) {
		ch += ((mode & 0x4) ? -1 : 1);
	    }
	} else if (mode == 0x4) {
	    ch -= GetDelta(info);
	}
    }
    /* Clear away extension bits, if any */
    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharLen --
1915
1916
1917
1918
1919
1920
1921

1922
1923
1924
1925
1926
1927
1928
 */

int
Tcl_UniCharIsControl(
    int ch)			/* Unicode character to test. */
{
    if (UNICODE_OUT_OF_RANGE(ch)) {

	ch &= 0x1FFFFF;
	if ((ch == 0xE0001) || ((ch >= 0xE0020) && (ch <= 0xE007F))) {
	    return 1;
	}
	if ((ch >= 0xF0000) && ((ch & 0xFFFF) <= 0xFFFD)) {
	    return 1;
	}







>







1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
 */

int
Tcl_UniCharIsControl(
    int ch)			/* Unicode character to test. */
{
    if (UNICODE_OUT_OF_RANGE(ch)) {
	/* Clear away extension bits, if any */
	ch &= 0x1FFFFF;
	if ((ch == 0xE0001) || ((ch >= 0xE0020) && (ch <= 0xE007F))) {
	    return 1;
	}
	if ((ch >= 0xF0000) && ((ch & 0xFFFF) <= 0xFFFD)) {
	    return 1;
	}