Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch dgp-read-chars Excluding Merge-Ins
This is equivalent to a diff from 279154c448 to 967f356dde
|
2014-03-09
| ||
| 21:55 | Mark io-35.18b test as knownBug check-in: 4fb211cd74 user: jan.nijtmans tags: core-8-5-branch | |
|
2014-03-08
| ||
| 00:21 | socket -async and gets/puts stall on windows (Ticket [336441ed59]) This is a change for a problem... Closed-Leaf check-in: 521b7229c4 user: andreask tags: win-sock-async-connect-race-fix | |
|
2014-02-28
| ||
| 19:01 | Bring over the ReadChars rewrite for integration into the other I/O work. check-in: 22a9f73877 user: dgp tags: dgp-read-bytes | |
| 18:28 | tidy up. Closed-Leaf check-in: 967f356dde user: dgp tags: dgp-read-chars | |
| 18:25 | More ReadChars rewriting. Test suite now passes. Note that this reform simplifies ReadChars a fair ... check-in: 8928ad3eb5 user: dgp tags: dgp-read-chars | |
|
2014-02-27
| ||
| 20:21 | Work in progress attempting a ReadChars rewrite. check-in: 914a1b6351 user: dgp tags: dgp-read-chars | |
|
2014-02-26
| ||
| 20:28 | merge 8.5 check-in: 27707adf73 user: dgp tags: dgp-read-bytes | |
| 17:47 | New tests covering INPUT_NEED_NL flag handling. One exposes a bug. check-in: 279154c448 user: dgp tags: core-8-5-branch | |
|
2014-02-24
| ||
| 17:25 | Do not reopen a win serial channel for serial detection. There are issues with some Bluetooth virtua... check-in: b6459ef66c user: oehhar tags: core-8-5-branch | |
Changes to generic/tclIO.c.
| ︙ | ︙ | |||
5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 |
* using the factor.
*/
dstNeeded = spaceLeft;
}
dst = objPtr->bytes + offset;
/*
* [Bug 1462248]: The cause of the crash reported in this bug is this:
*
* - ReadChars, called with a single buffer, with a incomplete
* multi-byte character at the end (only the first byte of it).
* - Encoding translation fails, asks for more data
* - Data is read, and eof is reached, TCL_ENCODING_END (TEE) is set.
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 |
* using the factor.
*/
dstNeeded = spaceLeft;
}
dst = objPtr->bytes + offset;
#if 1
/*
* This routine is burdened with satisfying several constraints.
* It cannot append more than 'charsToRead` chars onto objPtr.
* This is measured after encoding and translation transformations
* are completed. There is no precise number of src bytes that can
* be associated with the limit. Yet, when we are done, we must know
* precisely the number of src bytes that were consumed to produce
* the appended chars, so that all subsequent bytes are left in
* the buffers for future read operations.
*
* The consequence is that we have no choice but to implement a
* "trial and error" approach, where in general we may need to
* perform transformations and copies multiple times to achieve
* a consistent set of results. This takes the shape of a loop.
*/
int dstLimit = dstNeeded + 1;
int savedFlags = statePtr->flags;
int savedIEFlags = statePtr->inputEncodingFlags;
Tcl_EncodingState savedState = statePtr->inputEncodingState;
while (1) {
int dstDecoded;
/*
* Perform the encoding transformation. Read no more than
* srcLen bytes, write no more than dstLimit bytes.
*/
int code = Tcl_ExternalToUtf(NULL, statePtr->encoding, src, srcLen,
statePtr->inputEncodingFlags & (bufPtr->nextPtr
? ~0 : ~TCL_ENCODING_END), &statePtr->inputEncodingState,
dst, dstLimit, &srcRead, &dstDecoded, &numChars);
/*
* Perform the translation transformation in place. Read no more
* than the dstDecoded bytes the encoding transformation actually
* produced. Capture the number of bytes written in dstWrote.
* Capture the number of bytes actually consumed in dstRead.
*/
dstWrote = dstRead = dstDecoded;
TranslateInputEOL(statePtr, dst, dst, &dstWrote, &dstRead);
if (dstRead < dstDecoded) {
/*
* The encoding transformation produced bytes that the
* translation transformation did not consume. Why did
* this happen?
*/
if (statePtr->inEofChar && dst[dstRead] == statePtr->inEofChar) {
/*
* 1) There's an eof char set on the channel, and
* we saw it and stopped translating at that point.
*
* NOTE the bizarre spec of TranslateInputEOL in this case.
* Clearly the eof char had to be read in order to account
* for the stopping, but the value of dstRead does not
* include it.
*
* Also rather bizarre, our caller can only notice an
* EOF condition if we return the value -1 as the number
* of chars read. This forces us to perform a 2-call
* dance where the first call can read all the chars
* up to the eof char, and the second call is solely
* for consuming the encoded eof char then pointed at
* by src so that we can return that magic -1 value.
* This seems really wasteful, especially since
* the first decoding pass of each call is likely to
* decode many bytes beyond that eof char that's all we
* care about.
*/
if (dstRead == 0) {
/*
* Curious choice in the eof char handling. We leave
* the eof char in the buffer. So, no need to compute
* a proper srcRead value. At this point, there
* are no chars before the eof char in the buffer.
*/
return -1;
}
{
/*
* There are chars leading the buffer before the eof
* char. Adjust the dstLimit so we go back and read
* only those and do not encounter the eof char this
* time.
*/
dstLimit = dstRead + TCL_UTF_MAX;
statePtr->flags = savedFlags;
statePtr->inputEncodingFlags = savedIEFlags;
statePtr->inputEncodingState = savedState;
continue;
}
}
/*
* 2) The other way to read fewer bytes than are decoded
* is when the final byte is \r and we're in a CRLF
* translation mode so we cannot decide whether to
* record \r or \n yet.
*/
assert(dstRead + 1 == dstDecoded);
assert(dst[dstRead] == '\r');
assert(statePtr->inputTranslation == TCL_TRANSLATE_CRLF);
if (dstWrote > 0) {
/*
* There are chars we can read before we hit the bare cr.
* Go back with a smaller dstLimit so we get them in the
* next pass, compute a matching srcRead, and don't end
* up back here in this call.
*/
dstLimit = dstRead + TCL_UTF_MAX;
statePtr->flags = savedFlags;
statePtr->inputEncodingFlags = savedIEFlags;
statePtr->inputEncodingState = savedState;
continue;
}
assert(dstWrote == 0);
assert(dstRead == 0);
assert(dstDecoded == 1);
/*
* We decoded only the bare cr, and we cannot read a
* translated char from that alone. We have to know what's
* next. So why do we only have the one decoded char?
*/
if (code != TCL_OK) {
char buffer[TCL_UTF_MAX + 2];
int read, decoded, count;
/*
* Didn't get everything the buffer could offer
*/
statePtr->flags = savedFlags;
statePtr->inputEncodingFlags = savedIEFlags;
statePtr->inputEncodingState = savedState;
Tcl_ExternalToUtf(NULL, statePtr->encoding, src, srcLen,
statePtr->inputEncodingFlags & (bufPtr->nextPtr
? ~0 : ~TCL_ENCODING_END), &statePtr->inputEncodingState,
buffer, TCL_UTF_MAX + 2, &read, &decoded, &count);
if (count == 2) {
if (buffer[1] == '\n') {
/* \r\n translate to \n */
dst[0] = '\n';
bufPtr->nextRemoved += read;
} else {
dst[0] = '\r';
bufPtr->nextRemoved += srcRead;
}
dst[1] = '\0';
statePtr->inputEncodingFlags &= ~TCL_ENCODING_START;
*offsetPtr += 1;
return 1;
}
} else if (statePtr->flags & CHANNEL_EOF) {
/*
* The bare \r is the only char and we will never read
* a subsequent char to make the determination.
*/
dst[0] = '\r';
bufPtr->nextRemoved = bufPtr->nextAdded;
*offsetPtr += 1;
return 1;
}
/* FALL THROUGH - get more data (dstWrote == 0) */
}
/*
* The translation transformation can only reduce the number
* of chars when it converts \r\n into \n. The reduction in
* the number of chars is the difference in bytes read and written.
*/
numChars -= (dstRead - dstWrote);
if (charsToRead > 0 && numChars > charsToRead) {
/*
* We read more chars than allowed. Reset limits to
* prevent that and try again.
*/
dstLimit = Tcl_UtfAtIndex(dst, charsToRead + 1) - dst;
statePtr->flags = savedFlags;
statePtr->inputEncodingFlags = savedIEFlags;
statePtr->inputEncodingState = savedState;
continue;
}
if (dstWrote == 0) {
/*
* We were not able to read any chars. Maybe there were
* not enough src bytes to decode into a char. Maybe
* a lone \r could not be translated (crlf mode). Need
* to combine any unused src bytes we have in the first
* buffer with subsequent bytes to try again.
*/
ChannelBuffer *nextPtr = bufPtr->nextPtr;
if (nextPtr == NULL) {
if (srcLen > 0) {
SetFlag(statePtr, CHANNEL_NEED_MORE_DATA);
}
return -1;
}
/*
* Space is made at the beginning of the buffer to copy the
* previous unused bytes there. Check first if the buffer we
* are using actually has enough space at its beginning for
* the data we are copying. Because if not we will write over
* the buffer management information, especially the 'nextPtr'.
*
* Note that the BUFFER_PADDING (See AllocChannelBuffer) is
* used to prevent exactly this situation. I.e. it should never
* happen. Therefore it is ok to panic should it happen despite
* the precautions.
*/
if (nextPtr->nextRemoved - srcLen < 0) {
Tcl_Panic("Buffer Underflow, BUFFER_PADDING not enough");
}
nextPtr->nextRemoved -= srcLen;
memcpy(RemovePoint(nextPtr), src, (size_t) srcLen);
RecycleBuffer(statePtr, bufPtr, 0);
statePtr->inQueueHead = nextPtr;
return ReadChars(statePtr, objPtr, charsToRead,
offsetPtr, factorPtr);
}
statePtr->inputEncodingFlags &= ~TCL_ENCODING_START;
bufPtr->nextRemoved += srcRead;
if (dstWrote > srcRead + 1) {
*factorPtr = dstWrote * UTF_EXPANSION_FACTOR / srcRead;
}
*offsetPtr += dstWrote;
return numChars;
}
#else
/*
* [Bug 1462248]: The cause of the crash reported in this bug is this:
*
* - ReadChars, called with a single buffer, with a incomplete
* multi-byte character at the end (only the first byte of it).
* - Encoding translation fails, asks for more data
* - Data is read, and eof is reached, TCL_ENCODING_END (TEE) is set.
|
| ︙ | ︙ | |||
5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 |
bufPtr->nextRemoved += srcRead;
if (dstWrote > srcRead + 1) {
*factorPtr = dstWrote * UTF_EXPANSION_FACTOR / srcRead;
}
*offsetPtr += dstWrote;
return numChars;
}
/*
*---------------------------------------------------------------------------
*
* TranslateInputEOL --
*
| > | 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 |
bufPtr->nextRemoved += srcRead;
if (dstWrote > srcRead + 1) {
*factorPtr = dstWrote * UTF_EXPANSION_FACTOR / srcRead;
}
*offsetPtr += dstWrote;
return numChars;
#endif
}
/*
*---------------------------------------------------------------------------
*
* TranslateInputEOL --
*
|
| ︙ | ︙ | |||
5657 5658 5659 5660 5661 5662 5663 |
srcEnd = srcStart + dstLen;
srcMax = srcStart + *srcLenPtr;
for ( ; src < srcEnd; ) {
if (*src == '\r') {
src++;
if (src >= srcMax) {
| | > | 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 |
srcEnd = srcStart + dstLen;
srcMax = srcStart + *srcLenPtr;
for ( ; src < srcEnd; ) {
if (*src == '\r') {
src++;
if (src >= srcMax) {
src--;
break;
} else if (*src == '\n') {
*dst++ = *src++;
} else {
*dst++ = '\r';
}
} else {
*dst++ = *src++;
|
| ︙ | ︙ |