Changes On Branch 967f356dde2f2d86
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch dgp-read-chars Excluding Merge-Ins

This is equivalent to a diff from 279154c448 to 967f356dde

2014-03-09
21:55
Mark io-35.18b test as knownBug check-in: 4fb211cd74 user: jan.nijtmans tags: core-8-5-branch
2014-03-08
00:21
socket -async and gets/puts stall on windows (Ticket [336441ed59]) This is a change for a problem... Closed-Leaf check-in: 521b7229c4 user: andreask tags: win-sock-async-connect-race-fix
2014-02-28
19:01
Bring over the ReadChars rewrite for integration into the other I/O work. check-in: 22a9f73877 user: dgp tags: dgp-read-bytes
18:28
tidy up. Closed-Leaf check-in: 967f356dde user: dgp tags: dgp-read-chars
18:25
More ReadChars rewriting. Test suite now passes. Note that this reform simplifies ReadChars a fair ... check-in: 8928ad3eb5 user: dgp tags: dgp-read-chars
2014-02-27
20:21
Work in progress attempting a ReadChars rewrite. check-in: 914a1b6351 user: dgp tags: dgp-read-chars
2014-02-26
20:28
merge 8.5 check-in: 27707adf73 user: dgp tags: dgp-read-bytes
17:47
New tests covering INPUT_NEED_NL flag handling. One exposes a bug. check-in: 279154c448 user: dgp tags: core-8-5-branch
2014-02-24
17:25
Do not reopen a win serial channel for serial detection. There are issues with some Bluetooth virtua... check-in: b6459ef66c user: oehhar tags: core-8-5-branch

Changes to generic/tclIO.c.
5368
5369
5370
5371
5372
5373
5374










































































































































































































































































5375
5376
5377
5378
5379
5380
5381
	 * using the factor.
	 */

	dstNeeded = spaceLeft;
    }
    dst = objPtr->bytes + offset;











































































































































































































































































    /*
     * [Bug 1462248]: The cause of the crash reported in this bug is this:
     *
     * - ReadChars, called with a single buffer, with a incomplete
     *	 multi-byte character at the end (only the first byte of it).
     * - Encoding translation fails, asks for more data
     * - Data is read, and eof is reached, TCL_ENCODING_END (TEE) is set.







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
	 * using the factor.
	 */

	dstNeeded = spaceLeft;
    }
    dst = objPtr->bytes + offset;

#if 1

    /*
     * This routine is burdened with satisfying several constraints.
     * It cannot append more than 'charsToRead` chars onto objPtr.
     * This is measured after encoding and translation transformations
     * are completed.  There is no precise number of src bytes that can
     * be associated with the limit.  Yet, when we are done, we must know
     * precisely the number of src bytes that were consumed to produce
     * the appended chars, so that all subsequent bytes are left in
     * the buffers for future read operations.
     *
     * The consequence is that we have no choice but to implement a
     * "trial and error" approach, where in general we may need to
     * perform transformations and copies multiple times to achieve
     * a consistent set of results.  This takes the shape of a loop.
     */

    int dstLimit = dstNeeded + 1;
    int savedFlags = statePtr->flags;
    int savedIEFlags = statePtr->inputEncodingFlags;
    Tcl_EncodingState savedState = statePtr->inputEncodingState;

    while (1) {
	int dstDecoded;

	/*
	 * Perform the encoding transformation.  Read no more than
	 * srcLen bytes, write no more than dstLimit bytes.
	 */

	int code = Tcl_ExternalToUtf(NULL, statePtr->encoding, src, srcLen,
		statePtr->inputEncodingFlags & (bufPtr->nextPtr
		? ~0 : ~TCL_ENCODING_END), &statePtr->inputEncodingState,
		dst, dstLimit, &srcRead, &dstDecoded, &numChars);

	/*
	 * Perform the translation transformation in place.  Read no more
	 * than the dstDecoded bytes the encoding transformation actually
	 * produced.  Capture the number of bytes written in dstWrote.
	 * Capture the number of bytes actually consumed in dstRead.
	 */

	dstWrote = dstRead = dstDecoded;
	TranslateInputEOL(statePtr, dst, dst, &dstWrote, &dstRead);

	if (dstRead < dstDecoded) {

	    /*
	     * The encoding transformation produced bytes that the
	     * translation transformation did not consume.  Why did
	     * this happen?
	     */

	    if (statePtr->inEofChar && dst[dstRead] == statePtr->inEofChar) {
		/*
		 * 1) There's an eof char set on the channel, and
		 *    we saw it and stopped translating at that point.
		 *
		 * NOTE the bizarre spec of TranslateInputEOL in this case.
		 * Clearly the eof char had to be read in order to account
		 * for the stopping, but the value of dstRead does not
		 * include it.
		 *
		 * Also rather bizarre, our caller can only notice an
		 * EOF condition if we return the value -1 as the number
		 * of chars read.  This forces us to perform a 2-call
		 * dance where the first call can read all the chars
		 * up to the eof char, and the second call is solely
		 * for consuming the encoded eof char then pointed at
		 * by src so that we can return that magic -1 value.
		 * This seems really wasteful, especially since
		 * the first decoding pass of each call is likely to
		 * decode many bytes beyond that eof char that's all we
		 * care about.
		 */

		if (dstRead == 0) {
		    /*
		     * Curious choice in the eof char handling.  We leave
		     * the eof char in the buffer.  So, no need to compute
		     * a proper srcRead value.  At this point, there
		     * are no chars before the eof char in the buffer.
		     */
		    return -1;
		}

		{
		    /*
		     * There are chars leading the buffer before the eof
		     * char.  Adjust the dstLimit so we go back and read
		     * only those and do not encounter the eof char this
		     * time.
		     */

		    dstLimit = dstRead + TCL_UTF_MAX;
		    statePtr->flags = savedFlags;
		    statePtr->inputEncodingFlags = savedIEFlags;
		    statePtr->inputEncodingState = savedState;
		    continue;
		}
	    }

	    /*
	     * 2) The other way to read fewer bytes than are decoded
	     *    is when the final byte is \r and we're in a CRLF
	     *    translation mode so we cannot decide whether to
	     *	  record \r or \n yet.
	     */

	    assert(dstRead + 1 == dstDecoded);
	    assert(dst[dstRead] == '\r');
	    assert(statePtr->inputTranslation == TCL_TRANSLATE_CRLF);

	    if (dstWrote > 0) {
		/*
		 * There are chars we can read before we hit the bare cr.
		 * Go back with a smaller dstLimit so we get them in the
		 * next pass, compute a matching srcRead, and don't end
		 * up back here in this call.
		 */

		dstLimit = dstRead + TCL_UTF_MAX;
		statePtr->flags = savedFlags;
		statePtr->inputEncodingFlags = savedIEFlags;
		statePtr->inputEncodingState = savedState;
		continue;
	    }

	    assert(dstWrote == 0);
	    assert(dstRead == 0);
	    assert(dstDecoded == 1);

	    /*
	     * We decoded only the bare cr, and we cannot read a
	     * translated char from that alone.  We have to know what's
	     * next.  So why do we only have the one decoded char?
	     */

	    if (code != TCL_OK) {
		char buffer[TCL_UTF_MAX + 2];
		int read, decoded, count;

		/* 
		 * Didn't get everything the buffer could offer
		 */

		statePtr->flags = savedFlags;
		statePtr->inputEncodingFlags = savedIEFlags;
		statePtr->inputEncodingState = savedState;

		Tcl_ExternalToUtf(NULL, statePtr->encoding, src, srcLen,
		statePtr->inputEncodingFlags & (bufPtr->nextPtr
		? ~0 : ~TCL_ENCODING_END), &statePtr->inputEncodingState,
		buffer, TCL_UTF_MAX + 2, &read, &decoded, &count);

		if (count == 2) {
		    if (buffer[1] == '\n') {
			/* \r\n translate to \n */
			dst[0] = '\n';
			bufPtr->nextRemoved += read;
		    } else {
			dst[0] = '\r';
			bufPtr->nextRemoved += srcRead;
		    }

		    dst[1] = '\0';
		    statePtr->inputEncodingFlags &= ~TCL_ENCODING_START;

		    *offsetPtr += 1;
		    return 1;
		}

	    } else if (statePtr->flags & CHANNEL_EOF) {

		/*
		 * The bare \r is the only char and we will never read
		 * a subsequent char to make the determination.
		 */

		dst[0] = '\r';
		bufPtr->nextRemoved = bufPtr->nextAdded;
		*offsetPtr += 1;
		return 1;
	    }

	    /* FALL THROUGH - get more data (dstWrote == 0) */
	}

	/* 
	 * The translation transformation can only reduce the number
	 * of chars when it converts \r\n into \n.  The reduction in
	 * the number of chars is the difference in bytes read and written.
	 */

	numChars -= (dstRead - dstWrote);

	if (charsToRead > 0 && numChars > charsToRead) {

	    /* 
	     * We read more chars than allowed.  Reset limits to
	     * prevent that and try again.
	     */

	    dstLimit = Tcl_UtfAtIndex(dst, charsToRead + 1) - dst;
	    statePtr->flags = savedFlags;
	    statePtr->inputEncodingFlags = savedIEFlags;
	    statePtr->inputEncodingState = savedState;
	    continue;
	}

	if (dstWrote == 0) {

	    /*
	     * We were not able to read any chars.  Maybe there were
	     * not enough src bytes to decode into a char.  Maybe
	     * a lone \r could not be translated (crlf mode).  Need
	     * to combine any unused src bytes we have in the first
	     * buffer with subsequent bytes to try again.
	     */

	    ChannelBuffer *nextPtr = bufPtr->nextPtr;

	    if (nextPtr == NULL) {
		if (srcLen > 0) {
		    SetFlag(statePtr, CHANNEL_NEED_MORE_DATA);
		}
		return -1;
	    }

	    /*
	     * Space is made at the beginning of the buffer to copy the
	     * previous unused bytes there. Check first if the buffer we
	     * are using actually has enough space at its beginning for
	     * the data we are copying.  Because if not we will write over
	     * the buffer management information, especially the 'nextPtr'.
	     *
	     * Note that the BUFFER_PADDING (See AllocChannelBuffer) is
	     * used to prevent exactly this situation. I.e. it should never
	     * happen.  Therefore it is ok to panic should it happen despite
	     * the precautions.
	     */

	    if (nextPtr->nextRemoved - srcLen < 0) {
		Tcl_Panic("Buffer Underflow, BUFFER_PADDING not enough");
	    }

	    nextPtr->nextRemoved -= srcLen;
	    memcpy(RemovePoint(nextPtr), src, (size_t) srcLen);
	    RecycleBuffer(statePtr, bufPtr, 0);
	    statePtr->inQueueHead = nextPtr;
	    return ReadChars(statePtr, objPtr, charsToRead,
		    offsetPtr, factorPtr);
	}

	statePtr->inputEncodingFlags &= ~TCL_ENCODING_START;

	bufPtr->nextRemoved += srcRead;
	if (dstWrote > srcRead + 1) {
	    *factorPtr = dstWrote * UTF_EXPANSION_FACTOR / srcRead;
	}
	*offsetPtr += dstWrote;
	return numChars;
    }

#else
    /*
     * [Bug 1462248]: The cause of the crash reported in this bug is this:
     *
     * - ReadChars, called with a single buffer, with a incomplete
     *	 multi-byte character at the end (only the first byte of it).
     * - Encoding translation fails, asks for more data
     * - Data is read, and eof is reached, TCL_ENCODING_END (TEE) is set.
5556
5557
5558
5559
5560
5561
5562

5563
5564
5565
5566
5567
5568
5569

    bufPtr->nextRemoved += srcRead;
    if (dstWrote > srcRead + 1) {
	*factorPtr = dstWrote * UTF_EXPANSION_FACTOR / srcRead;
    }
    *offsetPtr += dstWrote;
    return numChars;

}

/*
 *---------------------------------------------------------------------------
 *
 * TranslateInputEOL --
 *







>







5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836

    bufPtr->nextRemoved += srcRead;
    if (dstWrote > srcRead + 1) {
	*factorPtr = dstWrote * UTF_EXPANSION_FACTOR / srcRead;
    }
    *offsetPtr += dstWrote;
    return numChars;
#endif
}

/*
 *---------------------------------------------------------------------------
 *
 * TranslateInputEOL --
 *
5657
5658
5659
5660
5661
5662
5663
5664

5665
5666
5667
5668
5669
5670
5671
	srcEnd = srcStart + dstLen;
	srcMax = srcStart + *srcLenPtr;

	for ( ; src < srcEnd; ) {
	    if (*src == '\r') {
		src++;
		if (src >= srcMax) {
		    SetFlag(statePtr, INPUT_NEED_NL);

		} else if (*src == '\n') {
		    *dst++ = *src++;
		} else {
		    *dst++ = '\r';
		}
	    } else {
		*dst++ = *src++;







|
>







5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
	srcEnd = srcStart + dstLen;
	srcMax = srcStart + *srcLenPtr;

	for ( ; src < srcEnd; ) {
	    if (*src == '\r') {
		src++;
		if (src >= srcMax) {
		    src--;
		    break;
		} else if (*src == '\n') {
		    *dst++ = *src++;
		} else {
		    *dst++ = '\r';
		}
	    } else {
		*dst++ = *src++;