Changes On Branch 749d917ed56628d2
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch tip-573 Through [749d917ed5] Excluding Merge-Ins

This is equivalent to a diff from eb525960d8 to 749d917ed5

2020-04-28
21:31
Merge 8.7 check-in: ae8e36391d user: jan.nijtmans tags: tip-573
19:44
merge 8.6 check-in: 8f2a784324 user: dgp tags: core-8-branch
18:29
Proposed fix. Closed-Leaf check-in: 5c063485fd user: dgp tags: bug-b6038e5bc8
14:37
Implementation for TIP #573: Surrogates are invalid check-in: 749d917ed5 user: jan.nijtmans tags: tip-573
07:52
Merge 8.7 check-in: ed1cd11bb5 user: jan.nijtmans tags: rfe-f443140a85
07:43
Merge 8.7 check-in: 6912f429be user: jan.nijtmans tags: trunk
07:41
Merge 8.6 check-in: eb525960d8 user: jan.nijtmans tags: core-8-branch
07:17
Merge 8.5 check-in: 5022b35087 user: jan.nijtmans tags: core-8-6-branch
2020-04-27
19:28
merge 8.6 check-in: f1da9ae4d0 user: dgp tags: core-8-branch

Changes to generic/tclParse.c.
864
865
866
867
868
869
870






871


872
873
874
875
876
877
878
879
880












881
882
883
884
885
886
887
864
865
866
867
868
869
870
871
872
873
874
875
876

877
878
879








880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898







+
+
+
+
+
+
-
+
+

-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+







    case 'u':
	count += ParseHex(p+1, (numBytes > 5) ? 4 : numBytes-2, &result);
	if (count == 2) {
	    /*
	     * No hexdigits -> This is just "u".
	     */
	    result = 'u';
#if TCL_UTF_MAX > 3
	} else if ((result & 0xF800) == 0xD800) {
	    result = 0xFFFD;
#else
	} else if ((result & 0xFC00) == 0xDC00) {
	    result = 0xFFFD;
	} else if (((result & 0xDC00) == 0xD800) && (count == 6)
	} else if ((result & 0xFC00) == 0xD800) {
	    if ((count == 6)
		    && (p[5] == '\\') && (p[6] == 'u') && (numBytes >= 10)) {
	    /* If high surrogate is immediately followed by a low surrogate
	     * escape, combine them into one character. */
	    int low;
	    int count2 = ParseHex(p+7, 4, &low);
	    if ((count2 == 4) && ((low & 0xDC00) == 0xDC00)) {
		result = ((result & 0x3FF)<<10 | (low & 0x3FF)) + 0x10000;
		count += count2 + 2;
	    }
		/* If high surrogate is immediately followed by a low surrogate
		 * escape, combine them into one character. */
		int low;
		int count2 = ParseHex(p+7, 4, &low);
		if ((count2 == 4) && ((low & 0xFC00) == 0xDC00)) {
		    result = ((result & 0x3FF)<<10 | (low & 0x3FF)) + 0x10000;
		    count += count2 + 2;
		    break;
		}
	    }
	    result = 0xFFFD;
#endif
	}
	break;
    case 'U':
	count += ParseHex(p+1, (numBytes > 9) ? 8 : numBytes-2, &result);
	if (count == 2) {
	    /*
	     * No hexdigits -> This is just "U".
Changes to generic/tclUtf.c.
489
490
491
492
493
494
495



496
497
498
499
500
501
502
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505







+
+
+







	    /*
	     * Three-byte-character lead byte followed by two trail bytes.
	     */

	    *chPtr = (((byte & 0x0F) << 12)
		    | ((src[1] & 0x3F) << 6) | (src[2] & 0x3F));
	    if (*chPtr > 0x7FF) {
		if ((*chPtr & 0xF800) == 0xD800) {
		    *chPtr = 0xFFFD;
		}
		return 3;
	    }
	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
585
586
587
588
589
590
591



592
593
594
595
596
597
598
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604







+
+
+







	    /*
	     * Three-byte-character lead byte followed by two trail bytes.
	     */

	    *chPtr = (((byte & 0x0F) << 12)
		    | ((src[1] & 0x3F) << 6) | (src[2] & 0x3F));
	    if (*chPtr > 0x7FF) {
		if ((*chPtr & 0xF800) == 0xD800) {
		    *chPtr = 0xFFFD;
		}
		return 3;
	    }
	}

	/*
	 * A three-byte-character lead-byte not followed by two trail-bytes
	 * represents itself.
Changes to tests/encoding.test.
34
35
36
37
38
39
40

41

42
43
44
45
46
47
48
34
35
36
37
38
39
40
41

42
43
44
45
46
47
48
49







+
-
+








# Some tests require the testencoding command
testConstraint testencoding [llength [info commands testencoding]]
testConstraint testbytestring [llength [info commands testbytestring]]
testConstraint teststringbytes [llength [info commands teststringbytes]]
testConstraint exec [llength [info commands exec]]
testConstraint testgetencpath [llength [info commands testgetencpath]]
testConstraint tip389 [expr {[string length [format %c 0x10000]] eq 2}]


# TclInitEncodingSubsystem is tested by the rest of this file
# TclFinalizeEncodingSubsystem is not currently tested

test encoding-1.1 {Tcl_GetEncoding: system encoding} -setup {
    set old [encoding system]
} -constraints {testencoding} -body {
    testencoding create foo [namespace origin toutf] [namespace origin fromutf]
321
322
323
324
325
326
327
328

329
330
331
332
333
334

335
336
337
338
339

340
341
342
343
344
345

346
347
348
349
350
351

352
353
354
355
356
357

358
359
360
361
362
363

364
365
366
367
368
369

370
371
372
373
374
375

376
377
378
379
380
381

382
383
384
385
386
387

388
389
390
391
392
393

394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410

411
412

413
414
415
416
417
418
419
420
421
422
423
424
425
426
427


428
429
430


431
432
433
434
435
436
437
322
323
324
325
326
327
328

329
330
331
332
333
334

335
336
337
338
339

340
341
342
343
344
345

346
347
348
349
350
351

352
353
354
355
356
357

358
359
360
361
362
363

364
365
366
367
368
369

370
371
372
373
374
375

376
377
378
379
380
381

382
383
384
385
386
387

388
389
390
391
392
393

394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410

411
412

413
414
415
416
417
418
419
420
421
422
423
424
425
426


427
428
429


430
431
432
433
434
435
436
437
438







-
+





-
+




-
+





-
+





-
+





-
+





-
+





-
+





-
+





-
+





-
+





-
+
















-
+

-
+













-
-
+
+

-
-
+
+







    binary scan [teststringbytes $y] H* z
    set z
} c080
test encoding-15.4 {UtfToUtfProc emoji character input} -body {
    set x \xED\xA0\xBD\xED\xB8\x82
    set y [encoding convertfrom utf-8 \xED\xA0\xBD\xED\xB8\x82]
    list [string length $x] $y
} -result "6 \U1F602"
} -result "6 \uFFFD\uFFFD"
test encoding-15.5 {UtfToUtfProc emoji character input} {
    set x \xF0\x9F\x98\x82
    set y [encoding convertfrom utf-8 \xF0\x9F\x98\x82]
    list [string length $x] $y
} "4 \U1F602"
test encoding-15.6 {UtfToUtfProc emoji character output} {
test encoding-15.6 {UtfToUtfProc emoji character output} tip389 {
    set x \uDE02\uD83D\uDE02\uD83D
    set y [encoding convertto utf-8 \uDE02\uD83D\uDE02\uD83D]
    binary scan $y H* z
    list [string length $y] $z
} {10 edb882f09f9882eda0bd}
} {10 efbfbdf09f9882efbfbd}
test encoding-15.7 {UtfToUtfProc emoji character output} {
    set x \uDE02\uD83D\uD83D
    set y [encoding convertto utf-8 \uDE02\uD83D\uD83D]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {3 9 edb882eda0bdeda0bd}
} {3 9 efbfbdefbfbdefbfbd}
test encoding-15.8 {UtfToUtfProc emoji character output} {
    set x \uDE02\uD83D\xE9
    set y [encoding convertto utf-8 \uDE02\uD83D\xE9]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {3 8 edb882eda0bdc3a9}
} {3 8 efbfbdefbfbdc3a9}
test encoding-15.9 {UtfToUtfProc emoji character output} {
    set x \uDE02\uD83DX
    set y [encoding convertto utf-8 \uDE02\uD83DX]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {3 7 edb882eda0bd58}
} {3 7 efbfbdefbfbd58}
test encoding-15.10 {UtfToUtfProc high surrogate character output} {
    set x \uDE02\xE9
    set y [encoding convertto utf-8 \uDE02\xE9]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {2 5 edb882c3a9}
} {2 5 efbfbdc3a9}
test encoding-15.11 {UtfToUtfProc low surrogate character output} {
    set x \uDA02\xE9
    set y [encoding convertto utf-8 \uDA02\xE9]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {2 5 eda882c3a9}
} {2 5 efbfbdc3a9}
test encoding-15.12 {UtfToUtfProc high surrogate character output} {
    set x \uDE02Y
    set y [encoding convertto utf-8 \uDE02Y]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {2 4 edb88259}
} {2 4 efbfbd59}
test encoding-15.13 {UtfToUtfProc low surrogate character output} {
    set x \uDA02Y
    set y [encoding convertto utf-8 \uDA02Y]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {2 4 eda88259}
} {2 4 efbfbd59}
test encoding-15.14 {UtfToUtfProc high surrogate character output} {
    set x \uDE02
    set y [encoding convertto utf-8 \uDE02]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {1 3 edb882}
} {1 3 efbfbd}
test encoding-15.15 {UtfToUtfProc low surrogate character output} {
    set x \uDA02
    set y [encoding convertto utf-8 \uDA02]
    binary scan $y H* z
    list [string length $x] [string length $y] $z
} {1 3 eda882}
} {1 3 efbfbd}
test encoding-15.16 {UtfToUtfProc emoji character output} {
    set x \U1F602
    set y [encoding convertto utf-8 \U1F602]
    binary scan $y H* z
    list [string length $y] $z
} {4 f09f9882}

test encoding-16.1 {Utf16ToUtfProc} -body {
    set val [encoding convertfrom utf-16 NN]
    list $val [format %x [scan $val %c]]
} -result "\u4E4E 4e4e"
test encoding-16.2 {Utf16ToUtfProc} -body {
    set val [encoding convertfrom utf-16 "\xD8\xD8\xDC\xDC"]
    list $val [format %x [scan $val %c]]
} -result "\U460DC 460dc"
test encoding-16.3 {Utf16ToUtfProc} -body {
    set val [encoding convertfrom utf-16 "\xDC\xDC"]
    set val [encoding convertfrom utf-16 "\xD4\xD4"]
    list $val [format %x [scan $val %c]]
} -result "\uDCDC dcdc"
} -result "\uD4D4 d4d4"
test encoding-16.4 {Ucs2ToUtfProc} -body {
    set val [encoding convertfrom ucs-2 NN]
    list $val [format %x [scan $val %c]]
} -result "\u4E4E 4e4e"
test encoding-16.4 {Ucs2ToUtfProc} -body {
    set val [encoding convertfrom ucs-2 "\xD8\xD8\xDC\xDC"]
    list $val [format %x [scan $val %c]]
} -result "\U460DC 460dc"

test encoding-17.1 {UtfToUtf16Proc} -body {
    encoding convertto utf-16 "\U460DC"
} -result "\xD8\xD8\xDC\xDC"
test encoding-17.2 {UtfToUtf16Proc} -body {
    encoding convertto utf-16 "\uDCDC"
} -result "\xDC\xDC"
    encoding convertto utf-16 "\uD4D4"
} -result "\xD4\xD4"
test encoding-17.3 {UtfToUtf16Proc} -body {
    encoding convertto utf-16 "\uD8D8"
} -result "\xD8\xD8"
    encoding convertto utf-16 "\uD0D0"
} -result "\xD0\xD0"
test encoding-17.4 {UtfToUcs2Proc} -body {
    encoding convertfrom utf-16 [encoding convertto ucs-2 "\U460DC"]
} -result "\uFFFD"

test encoding-18.1 {TableToUtfProc} {
} {}

Changes to tests/utf.test.
62
63
64
65
66
67
68
69

70
71
72

73
74
75
76
77
78
79
62
63
64
65
66
67
68

69
70
71

72
73
74
75
76
77
78
79







-
+


-
+







test utf-1.7.0 {Tcl_UniCharToUtf: 4 byte sequences} {fullutf Uesc testbytestring} {
    expr {"\U014E4E" eq [testbytestring "\xF0\x94\xB9\x8E"]}
} 1
test utf-1.7.1 {Tcl_UniCharToUtf: 4 byte sequences} {ucs2 Uesc testbytestring} {
    expr {"\U014E4E" eq [testbytestring "\xF0\x94\xB9\x8E"]}
} 0
test utf-1.8 {Tcl_UniCharToUtf: 3 byte sequence, high surrogate} testbytestring {
    expr {"\uD842" eq [testbytestring "\xED\xA1\x82"]}
    expr {"\uD842" eq [testbytestring "\xEF\xBF\xBD"]}
} 1
test utf-1.9 {Tcl_UniCharToUtf: 3 byte sequence, low surrogate} testbytestring {
    expr {"\uDC42" eq [testbytestring "\xED\xB1\x82"]}
    expr {"\uDC42" eq [testbytestring "\xEF\xBF\xBD"]}
} 1
test utf-1.10 {Tcl_UniCharToUtf: 3 byte sequence, high surrogate} testbytestring {
    expr {[format %c 0xD842] eq [testbytestring "\xED\xA1\x82"]}
} 1
test utf-1.11 {Tcl_UniCharToUtf: 3 byte sequence, low surrogate} testbytestring {
    expr {[format %c 0xDC42] eq [testbytestring "\xED\xB1\x82"]}
} 1