*
 *---------------------------------------------------------------------------
 */

/* Since TCL_ENCODING_MODIFIED is only used for utf-8/cesu-8 and
 * TCL_ENCODING_LE is only used for  utf-16/utf-32/ucs-2. re-use the same value */
#define TCL_ENCODING_LE		TCL_ENCODING_MODIFIED	/* Little-endian encoding */
#define ENCODING_UTF	0x200	/* For UTF-8 encoding, allow 4-byte output sequences */
#define ENCODING_INPUT	0x400 /* For UTF-8/CESU-8 encoding, means external -> internal */

void
TclInitEncodingSubsystem(void)
{
    Tcl_EncodingType type;
    TableEncodingData *dataPtr;
    unsigned size;

    tclIdentityEncoding = Tcl_CreateEncoding(&type);

    type.encodingName	= "utf-8";
    type.toUtfProc	= UtfToUtfProc;
    type.fromUtfProc	= UtfToUtfProc;
    type.freeProc	= NULL;
    type.nullSize	= 1;
    type.clientData	= INT2PTR(ENCODING_UTF);
    Tcl_CreateEncoding(&type);
    type.clientData	= INT2PTR(TCL_ENCODING_NOCOMPLAIN);
    type.encodingName	= "cesu-8";
    Tcl_CreateEncoding(&type);

    type.toUtfProc	= Utf16ToUtfProc;
    type.fromUtfProc    = UtfToUcs2Proc;

	srcLen = 0;
    } else if (srcLen < 0) {
	srcLen = encodingPtr->lengthProc(src);
    }

    flags |= TCL_ENCODING_START | TCL_ENCODING_END;
    if (encodingPtr->toUtfProc == UtfToUtfProc) {
	flags |= ENCODING_INPUT;
    }

    while (1) {
	result = encodingPtr->toUtfProc(encodingPtr->clientData, src, srcLen,
		flags, &state, dst, dstLen, &srcRead, &dstWrote, &dstChars);
	soFar = dst + dstWrote - Tcl_DStringValue(dstPtr);

	 * the actual \0 at the end of the destination buffer, we need to
	 * append it manually.  First make room for it...
	 */

	dstLen--;
    }
    if (encodingPtr->toUtfProc == UtfToUtfProc) {
	flags |= ENCODING_INPUT;
    }
    do {
	Tcl_EncodingState savedState = *statePtr;

	result = encodingPtr->toUtfProc(encodingPtr->clientData, src, srcLen,
		flags, statePtr, dst, dstLen, srcReadPtr, dstWrotePtr,
		dstCharsPtr);

int
Tcl_UtfToExternalDStringEx(
    Tcl_Encoding encoding,	/* The encoding for the converted string, or
				 * NULL for the default system encoding. */
    const char *src,		/* Source string in UTF-8. */
    int srcLen,			/* Source string length in bytes, or < 0 for
				 * strlen(). */
    int flags,			/* Conversion control flags. */
    Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
				 * converted string is stored. */
{
    char *dst;
    Tcl_EncodingState state;
    const Encoding *encodingPtr;
    int dstLen, result, soFar, srcRead, dstWrote, dstChars;

    }
    if (flags & TCL_ENCODING_CHAR_LIMIT) {
	charLimit = *dstCharsPtr;
    }

    dstStart = dst;
    flags |= PTR2INT(clientData);
    dstEnd = dst + dstLen - ((flags & ENCODING_UTF) ? TCL_UTF_MAX : 6);

    for (numChars = 0; src < srcEnd && numChars <= charLimit; numChars++) {
	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
	    /*
	     * If there is more string to follow, this will ensure that the
	     * last UTF-8 character in the source buffer hasn't been cut off.
	     */

	    result = TCL_CONVERT_MULTIBYTE;
	    break;
	}
	if (dst > dstEnd) {
	    result = TCL_CONVERT_NOSPACE;
	    break;
	}
	if (UCHAR(*src) < 0x80 && !((UCHAR(*src) == 0) && (flags & ENCODING_INPUT))) {
	    /*
	     * Copy 7bit characters, but skip null-bytes when we are in input
	     * mode, so that they get converted to 0xC080.
	     */

	    *dst++ = *src++;
	} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd)
		&& (UCHAR(src[1]) == 0x80) && (flags & ENCODING_UTF) && (!(flags & ENCODING_INPUT)
			|| ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
			|| (flags & ENCODING_FAILINDEX))) {
	    /*
	     * If in input mode, and -strict or -failindex is specified: This is an error.
	     */
	    if (flags & ENCODING_INPUT) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }

	    /*
	     * Convert 0xC080 to real nulls when we are in output mode, with or without '-strict'.
	     */
	    *dst++ = 0;
	    src += 2;
	} else if (!Tcl_UtfCharComplete(src, srcEnd - src)) {
	    /*
	     * Always check before using TclUtfToUCS4. Not doing can so
	     * cause it run beyond the end of the buffer! If we happen such an
	     * incomplete char its bytes are made to represent themselves
	     * unless the user has explicitly asked to be told.
	     */

	    if (flags & ENCODING_INPUT) {
		if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) {
		    result = TCL_CONVERT_MULTIBYTE;
		    break;
		}
	    if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
		ch = UCHAR(*src++);
	    } else {
		char chbuf[2];
		chbuf[0] = UCHAR(*src++); chbuf[1] = 0;
		TclUtfToUCS4(chbuf, &ch);
	    }
	    dst += Tcl_UniCharToUtf(ch, dst);
	} else {
	    int low;
	    const char *saveSrc = src;
	    size_t len = TclUtfToUCS4(src, &ch);
	    if ((len < 2) && (ch != 0) && (flags & ENCODING_INPUT)
		    && (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT))) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    src += len;
	    if (!(flags & ENCODING_UTF) && (ch > 0x3FF)) {
		if (ch > 0xFFFF) {
		    /* CESU-8 6-byte sequence for chars > U+FFFF */
		    ch -= 0x10000;
		    *dst++ = 0xED;
		    *dst++ = (char) (((ch >> 16) & 0x0F) | 0xA0);
		    *dst++ = (char) (((ch >> 10) & 0x3F) | 0x80);
		    ch = (ch & 0x0CFF) | 0xDC00;
		}
		goto cesu8;
	    } else if ((ch | 0x7FF) == 0xDFFF) {
		/*
		 * A surrogate character is detected, handle especially.
		 */

		if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) && (flags & ENCODING_UTF)) {
		    result = TCL_CONVERT_UNKNOWN;
		    src = saveSrc;
		    break;
		}
		low = ch;
		len = (src <= srcEnd-3) ? TclUtfToUCS4(src, &low) : 0;

		if (((low & ~0x3FF) != 0xDC00) || (ch & 0x400)) {

		    if (STOPONERROR) {
			result = TCL_CONVERT_UNKNOWN;
			src = saveSrc;
			break;
		    }
		cesu8:
		    *dst++ = (char) (((ch >> 12) | 0xE0) & 0xEF);
		    *dst++ = (char) (((ch >> 6) | 0x80) & 0xBF);
		    *dst++ = (char) ((ch | 0x80) & 0xBF);
		    continue;
		}
		src += len;
		dst += Tcl_UniCharToUtf(ch, dst);
		ch = low;
	    } else if (STOPONERROR && !(flags & ENCODING_INPUT) && (((ch  & ~0x7FF) == 0xD800))) {
		result = TCL_CONVERT_UNKNOWN;
		src = saveSrc;
		break;
	    } else if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
		    && (flags & ENCODING_INPUT) && ((ch  & ~0x7FF) == 0xD800)) {
		result = TCL_CONVERT_SYNTAX;
		src = saveSrc;
		break;
	    }
	    dst += Tcl_UniCharToUtf(ch, dst);
	}
    }

		break;
	    }
	    ch = toUnicode[byte][*((unsigned char *) src)];
	} else {
	    ch = pageZero[byte];
	}
	if ((ch == 0) && (byte != 0)) {
	    if ((flags & ENCODING_FAILINDEX)
		    || ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)) {
		result = TCL_CONVERT_SYNTAX;
		break;
	    }
	    if (prefixBytes[byte]) {
		src--;
	    }
	    ch = (Tcl_UniChar) byte;

    list [string length $y] $z
} {2 c480}
test encoding-15.24 {UtfToUtfProc CESU-8 bug [048dd20b4171c8da]} {
    set y [encoding convertto cesu-8 \u3FF]
    binary scan $y H* z
    list [string length $y] $z
} {2 cfbf}
test encoding-15.25 {UtfToUtfProc CESU-8} {
    encoding convertfrom cesu-8 \x00
} \x00
test encoding-15.26 {UtfToUtfProc CESU-8} {
    encoding convertfrom cesu-8 \xC0\x80
} \x00
test encoding-15.27 {UtfToUtfProc -strict CESU-8} {
    encoding convertfrom -strict cesu-8 \xC0\x80
} \x00
test encoding-15.28 {UtfToUtfProc -strict CESU-8} {
    encoding convertfrom -strict cesu-8 \xC0\x80
} \x00
test encoding-15.29 {UtfToUtfProc CESU-8} {
    encoding convertto cesu-8 \x00
} \xC0\x80
test encoding-15.30 {UtfToUtfProc -strict CESU-8} {
    encoding convertto -strict cesu-8 \x00
} \xC0\x80

test encoding-16.1 {Utf16ToUtfProc} -body {
    set val [encoding convertfrom utf-16 NN]
    list $val [format %x [scan $val %c]]
} -result "乎 4e4e"
test encoding-16.2 {Utf16ToUtfProc} -body {
    set val [encoding convertfrom utf-16 "\xD8\xD8\xDC\xDC"]

test encoding-18.5 {TableToUtfProc on invalid input with -failindex} -body {
	list [catch {encoding convertto -failindex pos jis0208 \\} res] $res $pos
} -result {0 {} 0}
test encoding-18.6 {TableToUtfProc on invalid input with -nocomplain} -body {
	list [catch {encoding convertto -nocomplain jis0208 \\} res] $res
} -result {0 !)}

test encoding-19.1 {TableFromUtfProc} -body {
    encoding convertfrom ascii AÁ
} -result AÁ
test encoding-19.2 {TableFromUtfProc} -body {
    encoding convertfrom -nocomplain ascii AÁ
} -result AÁ
test encoding-19.3 {TableFromUtfProc} -body {
    encoding convertfrom -strict ascii AÁ
} -returnCodes 1 -result {unexpected byte sequence starting at index 1: '\xC1'}
test encoding-19.4 {TableFromUtfProc} -body {
    list [encoding convertfrom -failindex idx ascii AÁ] [set idx]
} -result {A 1}
test encoding-19.4 {TableFromUtfProc} -body {
    list [encoding convertfrom -failindex idx -strict ascii AÁ] [set idx]
} -result {A 1}

test encoding-20.1 {TableFreefProc} {
} {}

test encoding-21.1 {EscapeToUtfProc} {
} {}

} -result \xED\xA0\x80
test encoding-24.39 {Try to generate invalid utf-8 with -strict} -body {
    encoding convertto -strict utf-8 \uD800
} -returnCodes 1 -result {unexpected character at index 0: 'U+00D800'}
test encoding-24.40 {Try to generate invalid utf-8 with -nocomplain} -body {
    encoding convertto -nocomplain utf-8 \uD800
} -result \xED\xA0\x80
test encoding-24.41 {Parse invalid utf-8 with -strict} -body {
    encoding convertfrom -strict utf-8 \xED\xA0\x80\xED\xB0\x80
} -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\xED'}

file delete [file join [temporaryDirectory] iso2022.txt]

#
# Begin jajp encoding round-trip conformity tests
#
proc foreach-jisx0208 {varName command} {