Index: generic/tcl.decls ================================================================== --- generic/tcl.decls +++ generic/tcl.decls @@ -2388,14 +2388,22 @@ } declare 694 { int Tcl_ListObjRange(Tcl_Interp *interp, Tcl_Obj *objPtr, Tcl_Size start, Tcl_Size end, Tcl_Obj **resultPtrPtr) } + +# TIP 726 +declare 695 { + const char *Tcl_UtfToNormalizedDString(Tcl_Interp *interp, + const char *bytes, Tcl_Size length, + Tcl_UnicodeNormalizationForm normForm, int profile, + Tcl_DString *dsPtr) +} # ----- BASELINE -- FOR -- 9.1.0 ----- # -declare 695 { +declare 696 { void TclUnusedStubEntry(void) } ############################################################################## Index: generic/tcl.h ================================================================== --- generic/tcl.h +++ generic/tcl.h @@ -2044,10 +2044,17 @@ #elif TCL_UTF_MAX == 3 && !defined(BUILD_tcl) typedef unsigned short Tcl_UniChar; #else # error "This TCL_UTF_MAX value is not supported" #endif + +/* + * Specifiers for Unicode normalization forms. + */ +typedef enum { + TCL_NFC, TCL_NFD, TCL_NFKC, TCL_NFKD +} Tcl_UnicodeNormalizationForm; /* *---------------------------------------------------------------------------- * TIP #59: The following structure is used in calls 'Tcl_RegisterConfig' to * provide the system with the embedded configuration data. Index: generic/tclDecls.h ================================================================== --- generic/tclDecls.h +++ generic/tclDecls.h @@ -1885,10 +1885,15 @@ /* 694 */ EXTERN int Tcl_ListObjRange(Tcl_Interp *interp, Tcl_Obj *objPtr, Tcl_Size start, Tcl_Size end, Tcl_Obj **resultPtrPtr); /* 695 */ +EXTERN const char * Tcl_UtfToNormalizedDString(Tcl_Interp *interp, + const char *bytes, Tcl_Size length, + Tcl_UnicodeNormalizationForm normForm, + int profile, Tcl_DString *dsPtr); +/* 696 */ EXTERN void TclUnusedStubEntry(void); typedef struct { const struct TclPlatStubs *tclPlatStubs; const struct TclIntStubs *tclIntStubs; @@ -2592,11 +2597,12 @@ int (*tcl_IsEmpty) (Tcl_Obj *obj); /* 690 */ const char * (*tcl_GetEncodingNameForUser) (Tcl_DString *bufPtr); /* 691 */ int (*tcl_ListObjReverse) (Tcl_Interp *interp, Tcl_Obj *objPtr, Tcl_Obj **resultPtrPtr); /* 692 */ int (*tcl_ListObjRepeat) (Tcl_Interp *interp, Tcl_Size repeatCount, Tcl_Size objc, Tcl_Obj *const objv[], Tcl_Obj **resultPtrPtr); /* 693 */ int (*tcl_ListObjRange) (Tcl_Interp *interp, Tcl_Obj *objPtr, Tcl_Size start, Tcl_Size end, Tcl_Obj **resultPtrPtr); /* 694 */ - void (*tclUnusedStubEntry) (void); /* 695 */ + const char * (*tcl_UtfToNormalizedDString) (Tcl_Interp *interp, const char *bytes, Tcl_Size length, Tcl_UnicodeNormalizationForm normForm, int profile, Tcl_DString *dsPtr); /* 695 */ + void (*tclUnusedStubEntry) (void); /* 696 */ } TclStubs; extern const TclStubs *tclStubsPtr; #ifdef __cplusplus @@ -3932,12 +3938,14 @@ (tclStubsPtr->tcl_ListObjReverse) /* 692 */ #define Tcl_ListObjRepeat \ (tclStubsPtr->tcl_ListObjRepeat) /* 693 */ #define Tcl_ListObjRange \ (tclStubsPtr->tcl_ListObjRange) /* 694 */ +#define Tcl_UtfToNormalizedDString \ + (tclStubsPtr->tcl_UtfToNormalizedDString) /* 695 */ #define TclUnusedStubEntry \ - (tclStubsPtr->tclUnusedStubEntry) /* 695 */ + (tclStubsPtr->tclUnusedStubEntry) /* 696 */ #endif /* defined(USE_TCL_STUBS) */ /* !END!: Do not edit above this line. */ Index: generic/tclEncoding.c ================================================================== --- generic/tclEncoding.c +++ generic/tclEncoding.c @@ -9,10 +9,11 @@ * this file, and for a DISCLAIMER OF ALL WARRANTIES. */ #include "tclInt.h" #include +#include "../utf8proc/utf8proc.h" /* Relative path to ignore system include */ typedef size_t (LengthProc)(const char *src); /* * The following data structure represents an encoding, which describes how to @@ -4635,12 +4636,96 @@ Tcl_NewStringObj(encodingProfiles[i].name, TCL_INDEX_NONE)); } Tcl_SetObjResult(interp, objPtr); } +/* + * Tcl_UtfToNormalizedDString -- + * + * Converts the passed string to a Unicode normalization form storing + * it in dsPtr. + * + * Results: + * Pointer to content of dsPtr on success, and NULL on error. + * + */ +const char * +Tcl_UtfToNormalizedDString( + Tcl_Interp *interp, /* Used for error messages. May be NULL */ + const char *bytes, /* Operand encoded in Tcl internal UTF8 */ + Tcl_Size length, /* Length bytes[], or -1 if NUL terminated */ + Tcl_UnicodeNormalizationForm normForm, /* TCL_{NFC,NFD,NFKC,NFKC} */ + int profile, /* TCL_ENCODING_PROFILE_{STRICT,REPLACE} */ + Tcl_DString *dsPtr) /* Converted output string in Tcl internal + UTF8 encoding. Init'ed by function */ +{ + Tcl_DStringInit(dsPtr); + + if (profile != TCL_ENCODING_PROFILE_REPLACE && + profile != TCL_ENCODING_PROFILE_STRICT) { + Tcl_SetObjResult(interp, + Tcl_ObjPrintf("Invalid value %d passed for encoding profile", + profile)); + } + + Tcl_Encoding encoding = Tcl_GetEncoding(interp, "utf-8"); + if (encoding == NULL) { + return NULL; + } + + int result; + Tcl_DString dsExt; + result = Tcl_UtfToExternalDStringEx(interp, encoding, bytes, length, + profile, &dsExt, NULL); + /* !!! dsExt needs to be freed even in case of error returns */ + + if (result == TCL_OK) { + utf8proc_uint8_t *normUtf8; + utf8proc_ssize_t normLength; + Tcl_Size dsLength = Tcl_DStringLength(&dsExt); + const utf8proc_uint8_t *dsStr = + (utf8proc_uint8_t *)Tcl_DStringValue(&dsExt); + utf8proc_option_t options = UTF8PROC_STABLE; + switch (normForm) { + case TCL_NFC: + options |= UTF8PROC_COMPOSE; + break; + case TCL_NFD: + options |= UTF8PROC_DECOMPOSE; + break; + case TCL_NFKC: + options |= UTF8PROC_COMPOSE|UTF8PROC_COMPAT; + break; + case TCL_NFKD: + options |= UTF8PROC_DECOMPOSE|UTF8PROC_COMPAT; + break; + } + normLength = utf8proc_map_custom(dsStr, dsLength, &normUtf8, + options, NULL, NULL); + + if (normLength < 0) { + const char *errorMsg = utf8proc_errmsg(normLength); + Tcl_SetObjResult( + interp, Tcl_NewStringObj( + errorMsg ? errorMsg : "Unicode normalization failed.", -1)); + result = TCL_ERROR; + } else { + /* Convert standard UTF8 to internal UTF8 */ + assert(normUtf8); + result = Tcl_ExternalToUtfDStringEx(interp, encoding, + (const char *)normUtf8, normLength, profile, dsPtr, NULL); + free(normUtf8); + } + } + + Tcl_DStringFree(&dsExt); + Tcl_FreeEncoding(encoding); + return result == TCL_OK ? Tcl_DStringValue(dsPtr) : NULL; +} + /* * Local Variables: * mode: c * c-basic-offset: 4 * fill-column: 78 * End: */ Index: generic/tclStubInit.c ================================================================== --- generic/tclStubInit.c +++ generic/tclStubInit.c @@ -1500,9 +1500,10 @@ Tcl_IsEmpty, /* 690 */ Tcl_GetEncodingNameForUser, /* 691 */ Tcl_ListObjReverse, /* 692 */ Tcl_ListObjRepeat, /* 693 */ Tcl_ListObjRange, /* 694 */ - TclUnusedStubEntry, /* 695 */ + Tcl_UtfToNormalizedDString, /* 695 */ + TclUnusedStubEntry, /* 696 */ }; /* !END!: Do not edit above this line. */ Index: unix/Makefile.in ================================================================== --- unix/Makefile.in +++ unix/Makefile.in @@ -239,10 +239,11 @@ ZLIB_DIR = ${COMPAT_DIR}/zlib ZLIB_INCLUDE = @ZLIB_INCLUDE@ TOMMATH_DIR = $(TOP_DIR)/libtommath TOMMATH_INCLUDE = @TOMMATH_INCLUDE@ +UTF8PROC_DIR = $(TOP_DIR)/utf8proc CC = @CC@ OBJEXT = @OBJEXT@ #CC = purify -best-effort @CC@ -DPURIFY @@ -345,10 +346,12 @@ bn_mp_signed_rsh.o \ bn_mp_to_ubin.o bn_mp_unpack.o \ bn_s_mp_toom_mul.o bn_s_mp_toom_sqr.o bn_mp_to_radix.o \ bn_mp_ubin_size.o bn_mp_xor.o bn_mp_zero.o bn_s_mp_add.o \ bn_s_mp_mul_digs.o bn_s_mp_sqr.o bn_s_mp_sub.o + +UTF8PROC_OBJS = utf8proc.o STUB_LIB_OBJS = tclStubLib.o \ tclStubCall.o \ tclStubLibTbl.o \ tclTomMathStubLib.o \ @@ -372,11 +375,11 @@ Zinffast.o Zinflate.o Zinftrees.o Ztrees.o Zuncompr.o Zzutil.o TCL_OBJS = ${GENERIC_OBJS} ${UNIX_OBJS} ${NOTIFY_OBJS} ${COMPAT_OBJS} \ ${OO_OBJS} @DL_OBJS@ @PLAT_OBJS@ -OBJS = ${TCL_OBJS} @DTRACE_OBJ@ @ZLIB_OBJS@ @TOMMATH_OBJS@ +OBJS = ${TCL_OBJS} ${UTF8PROC_OBJS} @DTRACE_OBJ@ @ZLIB_OBJS@ @TOMMATH_OBJS@ TCL_DECLS = \ $(GENERIC_DIR)/tcl.decls \ $(GENERIC_DIR)/tclInt.decls \ $(GENERIC_DIR)/tclOO.decls \ @@ -718,16 +721,20 @@ $(ZLIB_DIR)/inftrees.c \ $(ZLIB_DIR)/trees.c \ $(ZLIB_DIR)/uncompr.c \ $(ZLIB_DIR)/zutil.c +UTF8PROC_SRCS = \ + $(UTF8PROC_DIR)/utf8proc.c + # Note: don't include DL_SRCS or MAC_OSX_SRCS in SRCS: most of those files # won't compile on the current machine, and they will cause problems for # things like "make depend". SRCS = $(GENERIC_SRCS) $(UNIX_SRCS) $(NOTIFY_SRCS) \ - $(OO_SRCS) $(STUB_SRCS) @PLAT_SRCS@ @ZLIB_SRCS@ @TOMMATH_SRCS@ + $(OO_SRCS) $(UTF8PROC_SRCS) $(STUB_SRCS) \ + @PLAT_SRCS@ @ZLIB_SRCS@ @TOMMATH_SRCS@ ### # Tip 430 - ZipFS Modifications ### @@ -1869,10 +1876,13 @@ $(CC) -c $(CC_SWITCHES) $(TCL_LOCATIONS) $(UNIX_DIR)/tclUnixInit.c tclUnixCompat.o: $(UNIX_DIR)/tclUnixCompat.c $(CC) -c $(CC_SWITCHES) $(UNIX_DIR)/tclUnixCompat.c +utf8proc.o: $(UTF8PROC_DIR)/utf8proc.c + $(CC) -c $(CC_SWITCHES) $(UTF8PROC_DIR)/utf8proc.c + # The following are Mac OS X only sources: tclMacOSXBundle.o: $(MAC_OSX_DIR)/tclMacOSXBundle.c $(CC) -c $(CC_SWITCHES) $(MAC_OSX_DIR)/tclMacOSXBundle.c tclMacOSXFCmd.o: $(MAC_OSX_DIR)/tclMacOSXFCmd.c @@ -1885,11 +1895,11 @@ tclWinError.o: $(TOP_DIR)/win/tclWinError.c $(CC) -c $(CC_SWITCHES) $(TOP_DIR)/win/tclWinError.c # DTrace support -$(TCL_OBJS) $(STUB_LIB_OBJS) $(TCLSH_OBJS) $(TCLTEST_OBJS) $(XTTEST_OBJS) $(TOMMATH_OBJS): @DTRACE_HDR@ +$(TCL_OBJS) $(STUB_LIB_OBJS) $(TCLSH_OBJS) $(TCLTEST_OBJS) $(XTTEST_OBJS) $(TOMMATH_OBJS) $(UTF8PROC_OBJS): @DTRACE_HDR@ $(DTRACE_HDR): $(DTRACE_SRC) $(DTRACE) -h $(DTRACE_SWITCHES) -o $@ -s $(DTRACE_SRC) $(DTRACE_OBJ): $(DTRACE_SRC) $(TCL_OBJS) @@ -2362,10 +2372,12 @@ $(INSTALL_DATA_DIR) $(DISTDIR)/libtommath @echo cp -r $(TOP_DIR)/libtommath $(DISTDIR)/libtommath @( cd $(TOP_DIR)/libtommath; find . -type f -print ) \ | ( cd $(TOP_DIR)/libtommath ; xargs tar cf - ) \ | ( cd $(DISTDIR)/libtommath ; tar xfp - ) + $(INSTALL_DATA_DIR) $(DISTDIR)/utf8proc + $(DIST_INSTALL_DATA) $(UTF8PROC_DIR)/*.[ch] $(DISTDIR)/utf8proc $(INSTALL_DATA_DIR) $(DISTDIR)/tests $(DIST_INSTALL_DATA) $(TOP_DIR)/license.terms $(DISTDIR)/tests $(DIST_INSTALL_DATA) $(TOP_DIR)/tests/*.test $(TOP_DIR)/tests/README \ $(TOP_DIR)/tests/*.bench $(TOP_DIR)/tests/*.tar.gz \ $(TOP_DIR)/tests/httpd $(TOP_DIR)/tests/*.tcl \ Index: win/Makefile.in ================================================================== --- win/Makefile.in +++ win/Makefile.in @@ -108,10 +108,11 @@ PKGS_DIR = $(TOP_DIR)/pkgs TOOL_DIR = $(TOP_DIR)/tools ZLIB_DIR = $(COMPAT_DIR)/zlib MINIZIP_DIR = $(ZLIB_DIR)/contrib/minizip TOMMATH_DIR = $(TOP_DIR)/libtommath +UTF8PROC_DIR = $(TOP_DIR)/utf8proc # Converts a POSIX path to a Windows native path. CYGPATH = @CYGPATH@ libdir_native = $(shell $(CYGPATH) '$(libdir)') @@ -448,10 +449,11 @@ bn_s_mp_sqr.${OBJEXT} \ bn_s_mp_sub.${OBJEXT} \ bn_s_mp_toom_mul.${OBJEXT} \ bn_s_mp_toom_sqr.${OBJEXT} +UTF8PROC_OBJS = utf8proc.${OBJEXT} WIN_OBJS = \ tclWin32Dll.$(OBJEXT) \ tclWinChan.$(OBJEXT) \ tclWinConsole.$(OBJEXT) \ @@ -492,11 +494,11 @@ inftrees.$(OBJEXT) \ trees.$(OBJEXT) \ uncompr.$(OBJEXT) \ zutil.$(OBJEXT) -TCL_OBJS = ${GENERIC_OBJS} ${WIN_OBJS} @ZLIB_OBJS@ @TOMMATH_OBJS@ +TCL_OBJS = ${GENERIC_OBJS} ${UTF8PROC_OBJS} ${WIN_OBJS} @ZLIB_OBJS@ @TOMMATH_OBJS@ TCL_DOCS = "$(ROOT_DIR_NATIVE)"/doc/*.[13n] all: binaries libraries doc packages @@ -682,10 +684,12 @@ # TIP #430, ZipFS Support tclZipfs.${OBJEXT}: $(GENERIC_DIR)/tclZipfs.c $(CC) -c $(CC_SWITCHES) -DBUILD_tcl \ $(ZLIB_INCLUDE) -I$(MINIZIP_DIR_NATIVE) @DEPARG@ $(CC_OBJNAME) +utf8proc.${OBJEXT}: $(UTF8PROC_DIR)/utf8proc.c + $(CC) -c $(CC_SWITCHES) -DUNICODE -D_UNICODE -DUTF8PROC_EXPORTS @DEPARG@ $(CC_OBJNAME) # TIP #59, embedding of configuration information into the binary library. # # Part of Tcl's configuration information are the paths where it was installed # and where it will look for its libraries (which can be different). We derive Index: win/makefile.vc ================================================================== --- win/makefile.vc +++ win/makefile.vc @@ -222,10 +222,12 @@ TCLDDELIB = $(OUT_DIR)\$(TCLDDELIBNAME) TCLTEST = $(OUT_DIR)\$(PROJECT)test$(VERSION)$(SUFX:t=).exe TCLTESTRAW = $(TCLTEST:.exe=-raw.exe) +UTF8PROCDIR = $(ROOT)\utf8proc + TCLSHOBJS = \ $(TMP_DIR)\tclAppInit.obj \ $(TMP_DIR)\tclsh.res TCLTESTOBJS = \ @@ -333,11 +335,12 @@ $(TMP_DIR)\tclTrace.obj \ $(TMP_DIR)\tclUtf.obj \ $(TMP_DIR)\tclUtil.obj \ $(TMP_DIR)\tclVar.obj \ $(TMP_DIR)\tclZipfs.obj \ - $(TMP_DIR)\tclZlib.obj + $(TMP_DIR)\tclZlib.obj \ + $(TMP_DIR)\utf8proc.obj \ !if $(STATIC_BUILD) ZLIBOBJS = \ $(TMP_DIR)\adler32.obj \ $(TMP_DIR)\compress.obj \ @@ -1021,10 +1024,15 @@ # Implicit rules that are not covered by the common ones defined in # rules.vc. A limitation exists with nmake that requires that # source directory can not contain spaces in the path. This an # absolute. #--------------------------------------------------------------------- + +{$(UTF8PROCDIR)}.c{$(TMP_DIR)}.obj:: + $(cc32) $(pkgcflags) -DUTF8PROC_EXPORTS -Fo$(TMP_DIR)\ @<< +$< +<< {$(TOMMATHDIR)}.c{$(TMP_DIR)}.obj:: $(cc32) $(pkgcflags) -Fo$(TMP_DIR)\ @<< $< <<