Changes On Branch c58db5881eb80f9e
Not logged in
Home Timeline Branches Tags Tickets Wiki Login
Patch Side-by-Side Diff

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch bug-bd1a60eb9c Excluding Merge-Ins

This is equivalent to a diff from 26aadbbd1f to c58db5881e

2023-02-14
21:42
Fix for [bd1a60eb9c]: convertfrom utf-8 strict mode allows surrogates in input sequence check-in: 84f30c5d4e user: jan.nijtmans tags: core-8-branch
20:50
Complete fix for [bd1a60eb9c]. Also fix a bug in the tableencoding. With testcases. Closed-Leaf check-in: c58db5881e user: jan.nijtmans tags: bug-bd1a60eb9c
10:43
Fix for [b8f575aa2398b0e4] and [154ed7ce564a7b4c], double-[read]/[gets] problem. Partial-read functi... Closed-Leaf check-in: ecd7aaec80 user: pooryorick tags: py-b8f575aa23-nopartial
07:29
Make a start fixing [bd1a60eb9c]. WIP check-in: 15fba8ddec user: jan.nijtmans tags: bug-bd1a60eb9c
2023-02-13
14:06
merge 8.7 check-in: 49849c85bb user: dgp tags: core-8-7-b1-rc
10:12
Merge 8.7. Mark (new) testcase encoding-16.15 as "knownBug": still needs a fix. check-in: a155df4adc user: jan.nijtmans tags: trunk, main
08:39
Fix for [10c2c17c32]: UTF-LE32 encoder mapping of surrogates. With testcases (both for utf-32 and ut... check-in: 26aadbbd1f user: jan.nijtmans tags: core-8-branch
2023-02-12
00:49
Proposed fix (and testcases) for [4a7397e0b3]: Tcl 9: fcopy with -strictencoding 1 UTF-8 channels check-in: 8b772479c3 user: jan.nijtmans tags: core-8-branch
2023-02-09
19:52
Add 4 more testcases, showing that the same bug is present in utf-16 as well. Also fix the bug (real... Closed-Leaf check-in: 0ecd1a4f6d user: jan.nijtmans tags: bug-10c2c17c32

Changes to generic/tclEncoding.c.
︙
515
516
517
518
519
520
521
522


523
524
525
526
527
528
529

515
516
517
518
519
520
521

522
523
524
525
526
527
528
529
530








-
+
+








 *
 *---------------------------------------------------------------------------
 */

/* Since TCL_ENCODING_MODIFIED is only used for utf-8/cesu-8 and
 * TCL_ENCODING_LE is only used for  utf-16/utf-32/ucs-2. re-use the same value */
#define TCL_ENCODING_LE		TCL_ENCODING_MODIFIED	/* Little-endian encoding */
#define TCL_ENCODING_UTF	0x200	/* For UTF-8 encoding, allow 4-byte output sequences */
#define ENCODING_UTF	0x200	/* For UTF-8 encoding, allow 4-byte output sequences */
#define ENCODING_INPUT	0x400 /* For UTF-8/CESU-8 encoding, means external -> internal */

void
TclInitEncodingSubsystem(void)
{
    Tcl_EncodingType type;
    TableEncodingData *dataPtr;
    unsigned size;
︙
557
558
559
560
561
562
563
564

565
566
567
568
569
570
571

558
559
560
561
562
563
564

565
566
567
568
569
570
571
572








-
+








    tclIdentityEncoding = Tcl_CreateEncoding(&type);

    type.encodingName	= "utf-8";
    type.toUtfProc	= UtfToUtfProc;
    type.fromUtfProc	= UtfToUtfProc;
    type.freeProc	= NULL;
    type.nullSize	= 1;
    type.clientData	= INT2PTR(TCL_ENCODING_UTF);
    type.clientData	= INT2PTR(ENCODING_UTF);
    Tcl_CreateEncoding(&type);
    type.clientData	= INT2PTR(TCL_ENCODING_NOCOMPLAIN);
    type.encodingName	= "cesu-8";
    Tcl_CreateEncoding(&type);

    type.toUtfProc	= Utf16ToUtfProc;
    type.fromUtfProc    = UtfToUcs2Proc;
︙
1234
1235
1236
1237
1238
1239
1240
1241

1242
1243
1244
1245
1246
1247
1248

1235
1236
1237
1238
1239
1240
1241

1242
1243
1244
1245
1246
1247
1248
1249








-
+








	srcLen = 0;
    } else if (srcLen < 0) {
	srcLen = encodingPtr->lengthProc(src);
    }

    flags |= TCL_ENCODING_START | TCL_ENCODING_END;
    if (encodingPtr->toUtfProc == UtfToUtfProc) {
	flags |= TCL_ENCODING_MODIFIED | TCL_ENCODING_UTF;
	flags |= ENCODING_INPUT;
    }

    while (1) {
	result = encodingPtr->toUtfProc(encodingPtr->clientData, src, srcLen,
		flags, &state, dst, dstLen, &srcRead, &dstWrote, &dstChars);
	soFar = dst + dstWrote - Tcl_DStringValue(dstPtr);
︙
1351
1352
1353
1354
1355
1356
1357
1358

1359
1360
1361
1362
1363
1364
1365

1352
1353
1354
1355
1356
1357
1358

1359
1360
1361
1362
1363
1364
1365
1366








-
+








	 * the actual \0 at the end of the destination buffer, we need to
	 * append it manually.  First make room for it...
	 */

	dstLen--;
    }
    if (encodingPtr->toUtfProc == UtfToUtfProc) {
	flags |= TCL_ENCODING_MODIFIED | TCL_ENCODING_UTF;
	flags |= ENCODING_INPUT;
    }
    do {
	Tcl_EncodingState savedState = *statePtr;

	result = encodingPtr->toUtfProc(encodingPtr->clientData, src, srcLen,
		flags, statePtr, dst, dstLen, srcReadPtr, dstWrotePtr,
		dstCharsPtr);
︙
1446
1447
1448
1449
1450
1451
1452
1453

1454
1455
1456
1457
1458
1459
1460

1447
1448
1449
1450
1451
1452
1453

1454
1455
1456
1457
1458
1459
1460
1461








-
+








int
Tcl_UtfToExternalDStringEx(
    Tcl_Encoding encoding,	/* The encoding for the converted string, or
				 * NULL for the default system encoding. */
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes, or < 0 for
				 * strlen(). */
    int flags,	/* Conversion control flags. */
    int flags,			/* Conversion control flags. */
    Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
				 * converted string is stored. */
{
    char *dst;
    Tcl_EncodingState state;
    const Encoding *encodingPtr;
    int dstLen, result, soFar, srcRead, dstWrote, dstChars;
︙
2359
2360
2361
2362
2363
2364
2365
2366

2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382

2383
2384
2385
2386
2387
2388
2389

2390


2391
2392
2393
2394

2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412

2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432

2433
2434
2435
2436
2437
2438

2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452





2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472

2473
2474
2475
2476
2477

2478
2479
2480
2481
2482
2483
2484

2360
2361
2362
2363
2364
2365
2366

2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382

2383
2384
2385
2386
2387
2388
2389
2390
2391

2392
2393
2394
2395
2396

2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414

2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434

2435
2436
2437
2438
2439
2440

2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479

2480
2481
2482
2483
2484

2485
2486
2487
2488
2489
2490
2491
2492








-
+















-
+







+
-
+
+



-
+

















-
+



















-
+





-
+














+
+
+
+
+



















-
+




-
+








    }
    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }

    dstStart = dst;
    flags |= PTR2INT(clientData);
    dstEnd = dst + dstLen - ((flags & TCL_ENCODING_UTF) ? TCL_UTF_MAX : 6);
    dstEnd = dst + dstLen - ((flags & ENCODING_UTF) ? TCL_UTF_MAX : 6);

    for (numChars = 0; src < srcEnd && numChars <= charLimit; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	if (UCHAR(*src) < 0x80 && !((UCHAR(*src) == 0) && (flags & TCL_ENCODING_MODIFIED))) {
	if (UCHAR(*src) < 0x80 && !((UCHAR(*src) == 0) && (flags & ENCODING_INPUT))) {
	    /*
	     * Copy 7bit characters, but skip null-bytes when we are in input
	     * mode, so that they get converted to 0xC080.
	     */

	    *dst++ = *src++;
	} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd)
		&& (UCHAR(src[1]) == 0x80) && (flags & ENCODING_UTF) && (!(flags & ENCODING_INPUT)
		&& (UCHAR(src[1]) == 0x80) && (!(flags & TCL_ENCODING_MODIFIED) || ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX))) {
			|| ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
			|| (flags & ENCODING_FAILINDEX))) {
	    /*
	     * If in input mode, and -strict or -failindex is specified: This is an error.
	     */
	    if (flags & TCL_ENCODING_MODIFIED) {
	    if (flags & ENCODING_INPUT) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }

	    /*
	     * Convert 0xC080 to real nulls when we are in output mode, with or without '-strict'.
	     */
	    *dst++ = 0;
	    src += 2;
	} else if (!Tcl_UtfCharComplete(src, srcEnd - src)) {
	    /*
	     * Always check before using TclUtfToUCS4. Not doing can so
	     * cause it run beyond the end of the buffer! If we happen such an
	     * incomplete char its bytes are made to represent themselves
	     * unless the user has explicitly asked to be told.
	     */

	    if (flags & TCL_ENCODING_MODIFIED) {
	    if (flags & ENCODING_INPUT) {
		if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) {
		    result = TCL_CONVERT_MULTIBYTE;
		    break;
		}
	    if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
		ch = UCHAR(*src++);
	    } else {
		char chbuf[2];
		chbuf[0] = UCHAR(*src++); chbuf[1] = 0;
		TclUtfToUCS4(chbuf, &ch);
	    }
	    dst += Tcl_UniCharToUtf(ch, dst);
	} else {
	    int low;
	    const char *saveSrc = src;
	    size_t len = TclUtfToUCS4(src, &ch);
	    if ((len < 2) && (ch != 0) && (flags & TCL_ENCODING_MODIFIED)
	    if ((len < 2) && (ch != 0) && (flags & ENCODING_INPUT)
		    && (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT))) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    src += len;
	    if (!(flags & TCL_ENCODING_UTF) && (ch > 0x3FF)) {
	    if (!(flags & ENCODING_UTF) && (ch > 0x3FF)) {
		if (ch > 0xFFFF) {
		    /* CESU-8 6-byte sequence for chars > U+FFFF */
		    ch -= 0x10000;
		    *dst++ = 0xED;
		    *dst++ = (char) (((ch >> 16) & 0x0F) | 0xA0);
		    *dst++ = (char) (((ch >> 10) & 0x3F) | 0x80);
		    ch = (ch & 0x0CFF) | 0xDC00;
		}
		goto cesu8;
	    } else if ((ch | 0x7FF) == 0xDFFF) {
		/*
		 * A surrogate character is detected, handle especially.
		 */

		if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) && (flags & ENCODING_UTF)) {
		    result = TCL_CONVERT_UNKNOWN;
		    src = saveSrc;
		    break;
		}
		low = ch;
		len = (src <= srcEnd-3) ? TclUtfToUCS4(src, &low) : 0;

		if (((low & ~0x3FF) != 0xDC00) || (ch & 0x400)) {

		    if (STOPONERROR) {
			result = TCL_CONVERT_UNKNOWN;
			src = saveSrc;
			break;
		    }
		cesu8:
		    *dst++ = (char) (((ch >> 12) | 0xE0) & 0xEF);
		    *dst++ = (char) (((ch >> 6) | 0x80) & 0xBF);
		    *dst++ = (char) ((ch | 0x80) & 0xBF);
		    continue;
		}
		src += len;
		dst += Tcl_UniCharToUtf(ch, dst);
		ch = low;
	    } else if (STOPONERROR && !(flags & TCL_ENCODING_MODIFIED) && (((ch  & ~0x7FF) == 0xD800))) {
	    } else if (STOPONERROR && !(flags & ENCODING_INPUT) && (((ch  & ~0x7FF) == 0xD800))) {
		result = TCL_CONVERT_UNKNOWN;
		src = saveSrc;
		break;
	    } else if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
		    && (flags & TCL_ENCODING_MODIFIED) && ((ch  & ~0x7FF) == 0xD800)) {
		    && (flags & ENCODING_INPUT) && ((ch  & ~0x7FF) == 0xD800)) {
		result = TCL_CONVERT_SYNTAX;
		src = saveSrc;
		break;
	    }
	    dst += Tcl_UniCharToUtf(ch, dst);
	}
    }
︙
3112
3113
3114
3115
3116
3117
3118
3119


3120
3121
3122
3123
3124
3125
3126

3120
3121
3122
3123
3124
3125
3126

3127
3128
3129
3130
3131
3132
3133
3134
3135








-
+
+








		break;
	    }
	    ch = toUnicode[byte][*((unsigned char *) src)];
	} else {
	    ch = pageZero[byte];
	}
	if ((ch == 0) && (byte != 0)) {
	    if (STOPONERROR) {
	    if ((flags & ENCODING_FAILINDEX)
		    || ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    if (prefixBytes[byte]) {
		src--;
	    }
	    ch = (Tcl_UniChar) byte;
︙
Changes to tests/encoding.test.
︙
448
449
450
451
452
453
454


















455
456
457
458
459
460
461

448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








    list [string length $y] $z
} {2 c480}
test encoding-15.24 {UtfToUtfProc CESU-8 bug [048dd20b4171c8da]} {
    set y [encoding convertto cesu-8 \u3FF]
    binary scan $y H* z
    list [string length $y] $z
} {2 cfbf}
test encoding-15.25 {UtfToUtfProc CESU-8} {
    encoding convertfrom cesu-8 \x00
} \x00
test encoding-15.26 {UtfToUtfProc CESU-8} {
    encoding convertfrom cesu-8 \xC0\x80
} \x00
test encoding-15.27 {UtfToUtfProc -strict CESU-8} {
    encoding convertfrom -strict cesu-8 \xC0\x80
} \x00
test encoding-15.28 {UtfToUtfProc -strict CESU-8} {
    encoding convertfrom -strict cesu-8 \xC0\x80
} \x00
test encoding-15.29 {UtfToUtfProc CESU-8} {
    encoding convertto cesu-8 \x00
} \xC0\x80
test encoding-15.30 {UtfToUtfProc -strict CESU-8} {
    encoding convertto -strict cesu-8 \x00
} \xC0\x80

test encoding-16.1 {Utf16ToUtfProc} -body {
    set val [encoding convertfrom utf-16 NN]
    list $val [format %x [scan $val %c]]
} -result "乎 4e4e"
test encoding-16.2 {Utf16ToUtfProc} -body {
    set val [encoding convertfrom utf-16 "\xD8\xD8\xDC\xDC"]
︙
580
581
582
583
584
585
586
587
588















589
590
591
592
593
594
595

598
599
600
601
602
603
604


605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626








-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








test encoding-18.5 {TableToUtfProc on invalid input with -failindex} -body {
	list [catch {encoding convertto -failindex pos jis0208 \\} res] $res $pos
} -result {0 {} 0}
test encoding-18.6 {TableToUtfProc on invalid input with -nocomplain} -body {
	list [catch {encoding convertto -nocomplain jis0208 \\} res] $res
} -result {0 !)}

test encoding-19.1 {TableFromUtfProc} {
} {}
test encoding-19.1 {TableFromUtfProc} -body {
    encoding convertfrom ascii AÁ
} -result AÁ
test encoding-19.2 {TableFromUtfProc} -body {
    encoding convertfrom -nocomplain ascii AÁ
} -result AÁ
test encoding-19.3 {TableFromUtfProc} -body {
    encoding convertfrom -strict ascii AÁ
} -returnCodes 1 -result {unexpected byte sequence starting at index 1: '\xC1'}
test encoding-19.4 {TableFromUtfProc} -body {
    list [encoding convertfrom -failindex idx ascii AÁ] [set idx]
} -result {A 1}
test encoding-19.4 {TableFromUtfProc} -body {
    list [encoding convertfrom -failindex idx -strict ascii AÁ] [set idx]
} -result {A 1}

test encoding-20.1 {TableFreefProc} {
} {}

test encoding-21.1 {EscapeToUtfProc} {
} {}
︙
800
801
802
803
804
805
806



807
808
809
810
811
812
813

831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847








+
+
+








} -result \xED\xA0\x80
test encoding-24.39 {Try to generate invalid utf-8 with -strict} -body {
    encoding convertto -strict utf-8 \uD800
} -returnCodes 1 -result {unexpected character at index 0: 'U+00D800'}
test encoding-24.40 {Try to generate invalid utf-8 with -nocomplain} -body {
    encoding convertto -nocomplain utf-8 \uD800
} -result \xED\xA0\x80
test encoding-24.41 {Parse invalid utf-8 with -strict} -body {
    encoding convertfrom -strict utf-8 \xED\xA0\x80\xED\xB0\x80
} -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\xED'}

file delete [file join [temporaryDirectory] iso2022.txt]

#
# Begin jajp encoding round-trip conformity tests
#
proc foreach-jisx0208 {varName command} {
︙
This page was created in about 0.031s by Fossil version [1205ec86cb] 2025-04-30 16:57:32