Diff
Not logged in

Differences From Artifact [9457d8b6d7]:

To Artifact [ee720bf978]:


92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
92
93
94
95
96
97
98





99
100
101
102
103
104
105







-
-
-
-
-







};

/*
 * Functions used only in this module.
 */

static int		Invalid(unsigned char *src);

#define UCS4ToUpper Tcl_UniCharToUpper
#define UCS4ToLower Tcl_UniCharToLower
#define UCS4ToTitle Tcl_UniCharToTitle


/*
 *---------------------------------------------------------------------------
 *
 * TclUtfCount --
 *
 *	Find the number of bytes in the Utf character "ch".
167
168
169
170
171
172
173
174

175
176
177
178
179
180
181
162
163
164
165
166
167
168

169
170
171
172
173
174
175
176







-
+







static int
Invalid(
    unsigned char *src)	/* Points to lead byte of a UTF-8 byte sequence */
{
    unsigned char byte = *src;
    int index;

    if (byte % 0x04) {
    if ((byte & 0xC3) != 0xC0) {
	/* Only lead bytes 0xC0, 0xE0, 0xF0, 0xF4 need examination */
	return 0;
    }
    index = (byte - 0xC0) >> 1;
    if (src[1] < bounds[index] || src[1] > bounds[index+1]) {
	/* Out of bounds - report invalid. */
	return 1;
439
440
441
442
443
444
445
446

447
448
449
450
451
452
453
434
435
436
437
438
439
440

441
442
443
444
445
446
447
448







-
+







   0x2DC, 0x2122, 0x0161, 0x203A, 0x0153,   0x9D, 0x017E, 0x0178
};

#undef Tcl_UtfToUniChar
int
Tcl_UtfToUniChar(
    const char *src,	/* The UTF-8 string. */
    int *chPtr)/* Filled with the unsigned int represented by
    int *chPtr)/* Filled with the Unicode character represented by
				 * the UTF-8 string. */
{
    int byte;

    /*
     * Unroll 1 to 4 byte UTF-8 sequences.
     */
501
502
503
504
505
506
507
508

509
510
511
512
513
514
515
496
497
498
499
500
501
502

503
504
505
506
507
508
509
510







-
+







	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
	 */
    }
    else if (byte < 0xF8) {
    else if (byte < 0xF5) {
	if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80) && ((src[3] & 0xC0) == 0x80)) {
	    /*
	     * Four-byte-character lead byte followed by three trail bytes.
	     */
	    *chPtr = (((byte & 0x07) << 18) | ((src[1] & 0x3F) << 12)
		    | ((src[2] & 0x3F) << 6) | (src[3] & 0x3F));
	    if ((unsigned)(*chPtr - 0x10000) <= 0xFFFFF) {
526
527
528
529
530
531
532
533
534


535
536
537
538
539
540
541
521
522
523
524
525
526
527


528
529
530
531
532
533
534
535
536







-
-
+
+







    *chPtr = byte;
    return 1;
}

int
Tcl_UtfToChar16(
    const char *src,	/* The UTF-8 string. */
    unsigned short *chPtr)/* Filled with the unsigned short represented by
				 * the UTF-8 string. */
    unsigned short *chPtr)/* Filled with the Tcl_UniChar represented by
				 * the UTF-8 string. This could be a surrogate too. */
{
    unsigned short byte;

    /*
     * Unroll 1 to 4 byte UTF-8 sequences.
     */

600
601
602
603
604
605
606
607
608


609
610


611
612

613
614

615
616
617
618
619
620

621
622
623
624
625
626
627
595
596
597
598
599
600
601


602
603
604

605
606
607

608
609

610


611
612
613
614
615
616
617
618
619
620
621
622







-
-
+
+

-
+
+

-
+

-
+
-
-




+







	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
	 */
    }
    else if (byte < 0xF8) {
	if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80) && ((src[3] & 0xC0) == 0x80)) {
    else if (byte < 0xF5) {
	if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80)) {
	    /*
	     * Four-byte-character lead byte followed by three trail bytes.
	     * Four-byte-character lead byte followed by at least two trail bytes.
	     * We don't test the validity of 3th trail byte, see [ed29806ba]
	     */
	    unsigned short high = (((byte & 0x07) << 8) | ((src[1] & 0x3F) << 2)
	    Tcl_UniChar high = (((byte & 0x07) << 8) | ((src[1] & 0x3F) << 2)
		    | ((src[2] & 0x3F) >> 4)) - 0x40;
	    if (high >= 0x400) {
	    if (high < 0x400) {
		/* out of range, < 0x10000 or > 0x10FFFF */
	    } else {
		/* produce high surrogate, advance source pointer */
		*chPtr = 0xD800 + high;
		return 1;
	    }
	    /* out of range, < 0x10000 or > 0x10FFFF */
	}

	/*
	 * A four-byte-character lead-byte not followed by three trail-bytes
	 * represents itself.
	 */
    }
778
779
780
781
782
783
784
785

786
787
788
789
790
791
792
773
774
775
776
777
778
779

780
781
782
783
784
785
786
787







-
+








int
Tcl_UtfCharComplete(
    const char *src,		/* String to check if first few bytes contain
				 * a complete UTF-8 character. */
    int length)			/* Length of above string in bytes. */
{
    return length >= complete[(unsigned char)*src];
    return length >= complete[UCHAR(*src)];
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_NumUtfChars --
 *
822
823
824
825
826
827
828
829

830
831
832
833
834
835

836
837
838
839
840
841
842
817
818
819
820
821
822
823

824
825
826
827
828
829

830
831
832
833
834
835
836
837







-
+





-
+







    if (length < 0) {
	while (*src != '\0') {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	if (i < 0) i = INT_MAX; /* Bug [2738427] */
    } else {
	const char *endPtr = src + length - 4;
	const char *endPtr = src + length - TCL_UTF_MAX;

	while (src < endPtr) {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	endPtr += 4;
	endPtr += TCL_UTF_MAX;
	while ((src < endPtr) && Tcl_UtfCharComplete(src, endPtr - src)) {
	    src += TclUtfToUniChar(src, &ch);
	    i++;
	}
	if (src < endPtr) {
	    i += endPtr - src;
	}
963
964
965
966
967
968
969
970

971
972
973
974
975
976
977
958
959
960
961
962
963
964

965
966
967
968
969
970
971
972







-
+







	     * before the needs of the lead byte were satisfied.
	     * Let the (malformed) lead byte alone be a character
	     */
	    return src + 1;
	}
	next++;
    }
    if ((next == src + 1) || Invalid((unsigned char *)src)) {
    if (Invalid((unsigned char *)src)) {
	return src + 1;
    }
    return next;
}

/*
 *---------------------------------------------------------------------------
1072
1073
1074
1075
1076
1077
1078
1079

1080
1081
1082
1083
1084
1085
1086

1087



1088
1089
1090
1091
1092
1093
1094
1067
1068
1069
1070
1071
1072
1073

1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093







-
+







+

+
+
+







	     * and exit this loop before we run past the start of the string.
	     */
	    return fallback;
	}

	/* Continue the search backwards... */
	look--;
    } while (trailBytesSeen < 4);
    } while (trailBytesSeen < TCL_UTF_MAX);

    /*
     * We've seen TCL_UTF_MAX trail bytes, so we know there will not be a
     * properly formed byte sequence to find, and we can stop looking,
     * accepting the fallback (for TCL_UTF_MAX > 3) or just go back as
     * far as we can.
     */
#if TCL_UTF_MAX > 3
    return fallback;
#else
    return src - TCL_UTF_MAX;
#endif
}

/*
 *---------------------------------------------------------------------------
 *
 * Tcl_UniCharAtIndex --
 *
1237
1238
1239
1240
1241
1242
1243
1244

1245
1246
1247
1248
1249
1250
1251
1252

1253
1254
1255
1256
1257
1258
1259
1236
1237
1238
1239
1240
1241
1242

1243
1244
1245
1246
1247
1248
1249
1250

1251
1252
1253
1254
1255
1256
1257
1258







-
+







-
+







    /*
     * Iterate over the string until we hit the terminating null.
     */

    src = dst = str;
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	upChar = UCS4ToUpper(ch);
	upChar = Tcl_UniCharToUpper(ch);

	/*
	 * To keep badly formed Utf strings from getting inflated by the
	 * conversion (thereby causing a segfault), only copy the upper case
	 * char to dst if its size is <= the original char.
	 */

	if ((len < TclUtfCount(upChar)) || ((upChar & 0xF800) == 0xD800)) {
	if ((len < TclUtfCount(upChar)) || ((upChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(upChar, dst);
	}
	src += len;
    }
1290
1291
1292
1293
1294
1295
1296
1297

1298
1299
1300
1301
1302
1303
1304
1305

1306
1307
1308
1309
1310
1311
1312
1289
1290
1291
1292
1293
1294
1295

1296
1297
1298
1299
1300
1301
1302
1303

1304
1305
1306
1307
1308
1309
1310
1311







-
+







-
+







    /*
     * Iterate over the string until we hit the terminating null.
     */

    src = dst = str;
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	lowChar = UCS4ToLower(ch);
	lowChar = Tcl_UniCharToLower(ch);

	/*
	 * To keep badly formed Utf strings from getting inflated by the
	 * conversion (thereby causing a segfault), only copy the lower case
	 * char to dst if its size is <= the original char.
	 */

	if ((len < TclUtfCount(lowChar)) || ((lowChar & 0xF800) == 0xD800)) {
	if ((len < TclUtfCount(lowChar)) || ((lowChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(lowChar, dst);
	}
	src += len;
    }
1346
1347
1348
1349
1350
1351
1352
1353

1354
1355

1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368

1369
1370
1371

1372
1373
1374
1375
1376
1377
1378
1345
1346
1347
1348
1349
1350
1351

1352
1353

1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366

1367
1368
1369

1370
1371
1372
1373
1374
1375
1376
1377







-
+

-
+












-
+


-
+







     * characters until we get to a null.
     */

    src = dst = str;

    if (*src) {
	len = TclUtfToUCS4(src, &ch);
	titleChar = UCS4ToTitle(ch);
	titleChar = Tcl_UniCharToTitle(ch);

	if ((len < TclUtfCount(titleChar)) || ((titleChar & 0xF800) == 0xD800)) {
	if ((len < TclUtfCount(titleChar)) || ((titleChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(titleChar, dst);
	}
	src += len;
    }
    while (*src) {
	len = TclUtfToUCS4(src, &ch);
	lowChar = ch;
	/* Special exception for Georgian Asomtavruli chars, no titlecase. */
	if ((unsigned)(lowChar - 0x1C90) >= 0x30) {
	    lowChar = UCS4ToLower(lowChar);
	    lowChar = Tcl_UniCharToLower(lowChar);
	}

	if ((len < TclUtfCount(lowChar)) || ((lowChar & 0xF800) == 0xD800)) {
	if ((len < TclUtfCount(lowChar)) || ((lowChar & ~0x7FF) == 0xD800)) {
	    memmove(dst, src, len);
	    dst += len;
	} else {
	    dst += Tcl_UniCharToUtf(lowChar, dst);
	}
	src += len;
    }
1658
1659
1660
1661
1662
1663
1664

1665
1666
1667
1668
1669
1670
1671
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671







+







    if (!UNICODE_OUT_OF_RANGE(ch)) {
	int info = GetUniCharInfo(ch);

	if (GetCaseType(info) & 0x04) {
	    ch -= GetDelta(info);
	}
    }
    /* Clear away extension bits, if any */
    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharToLower --
1689
1690
1691
1692
1693
1694
1695

1696
1697
1698
1699
1700
1701
1702
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703







+







	int info = GetUniCharInfo(ch);
	int mode = GetCaseType(info);

	if ((mode & 0x02) && (mode != 0x7)) {
	    ch += GetDelta(info);
	}
    }
    /* Clear away extension bits, if any */
    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharToTitle --
1728
1729
1730
1731
1732
1733
1734

1735
1736
1737
1738
1739
1740
1741
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743







+







	    if (mode != 0x7) {
		ch += ((mode & 0x4) ? -1 : 1);
	    }
	} else if (mode == 0x4) {
	    ch -= GetDelta(info);
	}
    }
    /* Clear away extension bits, if any */
    return ch & 0x1FFFFF;
}

/*
 *----------------------------------------------------------------------
 *
 * Tcl_UniCharLen --
1915
1916
1917
1918
1919
1920
1921

1922
1923
1924
1925
1926
1927
1928
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931







+







 */

int
Tcl_UniCharIsControl(
    int ch)			/* Unicode character to test. */
{
    if (UNICODE_OUT_OF_RANGE(ch)) {
	/* Clear away extension bits, if any */
	ch &= 0x1FFFFF;
	if ((ch == 0xE0001) || ((ch >= 0xE0020) && (ch <= 0xE007F))) {
	    return 1;
	}
	if ((ch >= 0xF0000) && ((ch & 0xFFFF) <= 0xFFFD)) {
	    return 1;
	}