Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Initial implementation of user-defined Tcl aggregate functions. Testing and documentation to come. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | andygoth-tcl-function |
Files: | files | file ages | folders |
SHA3-256: |
e0689f05d1f8792d9024d8f5a8dbd2a9 |
User & Date: | andygoth 2019-01-30 17:30:58 |
Context
2019-01-31
| ||
05:25 | Add method argument to aggregate function invocations check-in: 4fc35d5e09 user: andygoth tags: andygoth-tcl-function | |
2019-01-30
| ||
17:30 | Initial implementation of user-defined Tcl aggregate functions. Testing and documentation to come. check-in: e0689f05d1 user: andygoth tags: andygoth-tcl-function | |
14:01 | Enhancements to the index_usage utility program. check-in: 19c739b4a8 user: drh tags: trunk | |
Changes
Changes to src/tclsqlite.c.
︙ | ︙ | |||
895 896 897 898 899 900 901 | Tcl_ListObjAppendElement(p->interp, pCmd, Tcl_NewStringObj(zB, nB)); Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT); Tcl_DecrRefCount(pCmd); return (atoi(Tcl_GetStringResult(p->interp))); } /* | | > > > | > > > > > > | > > > > | | 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 | Tcl_ListObjAppendElement(p->interp, pCmd, Tcl_NewStringObj(zB, nB)); Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT); Tcl_DecrRefCount(pCmd); return (atoi(Tcl_GetStringResult(p->interp))); } /* ** Utility function shared by the following functions: ** - tclSqlFuncScalar() ** - tclSqlFuncStep() ** - tclSqlFuncFinal() ** This function calls a user-defined TCL script to implement an SQL function. ** Before being invoked, additional arguments may be appended to the script. ** Optionally, the first argument may be a TCL object. All remaining arguments ** are TCL objects constructed from SQL values. On success, the return value ** is a pointer to the TCL object result. On failure, the return value is NULL, ** and an error message is stored in the SQL context. */ static Tcl_Obj *tclSqlFuncCall( sqlite3_context *context, Tcl_Obj *pState, int argc, sqlite3_value **argv ){ SqlFunc *p = sqlite3_user_data(context); Tcl_Obj *pCmd; int i; int rc; if( argc==0 && !pState ){ /* If there are no arguments to the function, call Tcl_EvalObjEx on the ** script object directly. This allows the TCL compiler to generate ** bytecode for the command on the first invocation and thus make ** subsequent invocations much faster. */ pCmd = p->pScript; Tcl_IncrRefCount(pCmd); rc = Tcl_EvalObjEx(p->interp, pCmd, 0); |
︙ | ︙ | |||
927 928 929 930 931 932 933 | ** of the list to tclCmdNameType, that alternate representation will ** be preserved and reused on the next invocation. */ Tcl_Obj **aArg; int nArg; if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); | | > > > > > > > > | 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 | ** of the list to tclCmdNameType, that alternate representation will ** be preserved and reused on the next invocation. */ Tcl_Obj **aArg; int nArg; if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); return 0; } pCmd = Tcl_NewListObj(nArg, aArg); Tcl_IncrRefCount(pCmd); if( pState ){ rc = Tcl_ListObjAppendElement(p->interp, pCmd, pState); if( rc ){ Tcl_DecrRefCount(pCmd); sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); return 0; } } for(i=0; i<argc; i++){ sqlite3_value *pIn = argv[i]; Tcl_Obj *pVal; /* Set pVal to contain the i'th column of this row. */ switch( sqlite3_value_type(pIn) ){ case SQLITE_BLOB: { |
︙ | ︙ | |||
970 971 972 973 974 975 976 | break; } } rc = Tcl_ListObjAppendElement(p->interp, pCmd, pVal); if( rc ){ Tcl_DecrRefCount(pCmd); sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); | | > | > > > > > > > > > > > > > > > | | > > > | | | 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 | break; } } rc = Tcl_ListObjAppendElement(p->interp, pCmd, pVal); if( rc ){ Tcl_DecrRefCount(pCmd); sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); return 0; } } if( !p->useEvalObjv ){ /* Tcl_EvalObjEx() will automatically call Tcl_EvalObjv() if pCmd ** is a list without a string representation. To prevent this from ** happening, make sure pCmd has a valid string representation */ Tcl_GetString(pCmd); } rc = Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT); Tcl_DecrRefCount(pCmd); } if( rc && rc!=TCL_RETURN ){ sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1); return 0; }else{ return Tcl_GetObjResult(p->interp); } } /* ** Utility function shared by the following functions: ** - tclSqlFuncScalar() ** - tclSqlFuncFinal() ** This function converts a TCL object result to an SQL value and stores it into ** the SQL context. If the TCL object result pointer is NULL, i.e. if the TCL ** script result code was not TCL_OK or TCL_RETURN, no action is taken. */ static void tclSqlFuncResult( sqlite3_context *context, Tcl_Obj *pVar ){ int n; u8 *data; const char *zType; char c; if( pVar ){ zType = pVar->typePtr ? pVar->typePtr->name : ""; c = zType[0]; if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ /* Only return a BLOB type if the Tcl variable is a bytearray and ** has no string representation. */ data = Tcl_GetByteArrayFromObj(pVar, &n); sqlite3_result_blob(context, data, n, SQLITE_TRANSIENT); }else if( c=='b' && strcmp(zType,"boolean")==0 ){ Tcl_GetIntFromObj(0, pVar, &n); |
︙ | ︙ | |||
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 | sqlite3_result_int64(context, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); sqlite3_result_text(context, (char *)data, n, SQLITE_TRANSIENT); } } } #ifndef SQLITE_OMIT_AUTHORIZATION /* ** This is the authentication function. It appends the authentication ** type code and the two arguments to zCmd[] then invokes the result ** on the interpreter. The reply is examined to determine if the ** authentication fails or succeeds. | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 | sqlite3_result_int64(context, v); }else{ data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n); sqlite3_result_text(context, (char *)data, n, SQLITE_TRANSIENT); } } } /* ** This address is distinct from all possible Tcl_Obj pointers. It is used to ** signify that an error occurred in tclSqlFuncStep() and that tclSqlFuncFinal() ** must not invoke the TCL script again. */ static Tcl_Obj tclSqlFuncError; /* ** Utility function shared by the following functions: ** - tclSqlFuncStep() ** - tclSqlFuncFinal() ** This function returns the address of a buffer consisting of a TCL object ** pointer. The caller is expected to modify this to point to the TCL object ** that will be used as the first argument to the next invocation of the TCL ** script implementing an SQL aggregate function. ** ** Implementation note: When called from tclSqlFuncFinal(), it would be slightly ** faster to pass 0 rather than sizeof(*ppState) as the second argument to ** sqlite3_aggregate_context(). This would allow the no-rows case to avoid ** allocating a buffer that will be deallocated as soon as tclSqlFuncFinal() ** returns. However, the added code complexity is not justified for a tiny bump ** in performance that would only be realized in a corner case, and it's already ** necessary to allocate an empty Tcl_Obj in this case, so be straightforward ** rather than clever. */ static Tcl_Obj **tclSqlFuncState(sqlite3_context *context){ Tcl_Obj **ppState = sqlite3_aggregate_context(context, sizeof(*ppState)); if( !*ppState ){ *ppState = Tcl_NewObj(); Tcl_IncrRefCount(*ppState); } return ppState; } /* ** This routine is called to evaluate a scalar SQL function implemented ** using TCL script. */ static void tclSqlFuncScalar( sqlite3_context *context, int argc, sqlite3_value **argv ){ tclSqlFuncResult(context, tclSqlFuncCall(context, NULL, argc, argv)); } /* ** This routine is called to evaluate an aggregate SQL function implemented ** using TCL script. It is invoked for each row in the group. The TCL result ** of each invocation becomes the first argument to the next invocation. The ** first argument of the first invocation is empty string. */ static void tclSqlFuncStep( sqlite3_context *context, int argc, sqlite3_value **argv ){ Tcl_Obj **ppState; Tcl_Obj *pResult; /* Even though this check was performed when the function was registered, it ** is necessary to double-check here in case -argcount was not used, meaning ** SQLite will allow the function to have any number of arguments, even zero. ** The presence or absence of arguments is how the TCL script can distinguish ** between being called as a step function or a final function. Thus, being ** passed zero arguments would create ambiguity. */ if( argc==0 ){ sqlite3_result_error(context, "insufficient arguments: all user-defined " "TCL aggregate functions must have at least one argument", -1); return; } ppState = tclSqlFuncState(context); pResult = tclSqlFuncCall(context, *ppState, argc, argv); if( pResult ){ /* If the step function succeeded, increment the new state reference count ** before decrementing the old state reference count. This sequence will ** deallocate the old state if it is a distinct TCL object from the new, but ** it will not deallocate the old state if the TCL script modified it in ** place and then returned it again. ** ** BUG: The above is not entirely true. The reference count of *ppState is ** nonzero when it is passed to Tcl_EvalObjEx(), which further increments ** the reference count because it places it in an argument variable. As a ** result, the TCL script sees a shared Tcl_Obj which cannot be modified in ** place. Better performance could be had if the reference count were zero ** when passed to Tcl_EvalObjEx(), but this is dangerous because the Tcl_Obj ** will have zero reference count despite being pointed to by *ppState and ** is therefore theoretically vulnerable to surprise deallocation. ** ** It is tempting to remove Tcl_IncrRefCount() from tclSqlFuncState() so ** that the Tcl_Obj have zero reference count, gambling that no uncontrolled ** code will be able to find and destroy the Tcl_Obj since the only pointer ** to it is here within the SQLite TCL binding. However, not even this ** solves the problem, at least not in TCL 8.7, since the original argument ** value is still preserved inside the call stack diagnostics and therefore ** cannot be modified in place. ** ** One way to fix this is to instead store the state data in a variable ** rather than pass it as a value, then pass the name of the variable to the ** script. This will complicate the scripts, increasing maintenance burden ** and incurring further performance penalties which may offset any supposed ** performance gains arising from avoiding copying. ** ** Another way to fix this is for SQLite to create custom TCL commands to ** read and write the value of the state Tcl_Obj. But again, this will make ** the scripts more complicated, with the same costs as above. ** ** The bottom line is that this bug is probably not worth fixing here, and ** it's up to TCL to find a better way to manage the lifetime of Tcl_Objs ** being held hostage by introspection systems that are not being used. */ Tcl_IncrRefCount(pResult); Tcl_DecrRefCount(*ppState); *ppState = pResult; }else{ /* If the step function had an error result, deallocate the TCL state object ** now and point to tclSqlFuncError so that tclSqlFuncFinal() will know not ** to invoke the TCL script again. */ Tcl_DecrRefCount(*ppState); *ppState = &tclSqlFuncError; } } /* ** This routine is called to evaluate an aggregate SQL function implemented ** using TCL script. It is invoked at the end of each group. The TCL result ** of the prior invocation (via tclSqlFuncStep()) is the first argument to this ** invocation, or empty string if there were no rows. */ static void tclSqlFuncFinal(sqlite3_context *context){ Tcl_Obj **ppState = tclSqlFuncState(context); if( *ppState!=&tclSqlFuncError ){ tclSqlFuncResult(context, tclSqlFuncCall(context, *ppState, 0, 0)); Tcl_DecrRefCount(*ppState); } } #ifndef SQLITE_OMIT_AUTHORIZATION /* ** This is the authentication function. It appends the authentication ** type code and the two arguments to zCmd[] then invokes the result ** on the interpreter. The reply is examined to determine if the ** authentication fails or succeeds. |
︙ | ︙ | |||
2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 | /* ** $db function NAME [-argcount N] [-deterministic] SCRIPT ** ** Create a new SQL function called NAME. Whenever that function is ** called, invoke SCRIPT to evaluate the function. */ case DB_FUNCTION: { int flags = SQLITE_UTF8; SqlFunc *pFunc; Tcl_Obj *pScript; char *zName; int nArg = -1; int i; if( objc<4 ){ | > | 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 | /* ** $db function NAME [-argcount N] [-deterministic] SCRIPT ** ** Create a new SQL function called NAME. Whenever that function is ** called, invoke SCRIPT to evaluate the function. */ case DB_FUNCTION: { enum {SCALAR, AGGREGATE} type = SCALAR; int flags = SQLITE_UTF8; SqlFunc *pFunc; Tcl_Obj *pScript; char *zName; int nArg = -1; int i; if( objc<4 ){ |
︙ | ︙ | |||
2664 2665 2666 2667 2668 2669 2670 2671 2672 | (char*)0); return TCL_ERROR; } i++; }else if( n>2 && strncmp(z, "-deterministic",n)==0 ){ flags |= SQLITE_DETERMINISTIC; }else{ Tcl_AppendResult(interp, "bad option \"", z, | > > > | > > > > > > | | > > > > | 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 | (char*)0); return TCL_ERROR; } i++; }else if( n>2 && strncmp(z, "-deterministic",n)==0 ){ flags |= SQLITE_DETERMINISTIC; }else if( n>2 && strncmp(z, "-aggregate",n)==0 ){ type = AGGREGATE; }else{ Tcl_AppendResult(interp, "bad option \"", z, "\": must be -argcount, -deterministic, or -aggregate", (char *)0 ); return TCL_ERROR; } } if( type==AGGREGATE && nArg==0 ){ Tcl_AppendResult(interp, "all user-defined TCL aggregate functions must " "have at least one argument", (char *)0); return TCL_ERROR; } pScript = objv[objc-1]; zName = Tcl_GetStringFromObj(objv[2], 0); pFunc = findSqlFunc(pDb, zName); if( pFunc==0 ) return TCL_ERROR; if( pFunc->pScript ){ Tcl_DecrRefCount(pFunc->pScript); } pFunc->pScript = pScript; Tcl_IncrRefCount(pScript); pFunc->useEvalObjv = safeToUseEvalObjv(interp, pScript); if( type==SCALAR ){ rc = sqlite3_create_function(pDb->db, zName, nArg, flags, pFunc, tclSqlFuncScalar, 0, 0); }else{ rc = sqlite3_create_function(pDb->db, zName, nArg, flags, pFunc, 0, tclSqlFuncStep, tclSqlFuncFinal); } if( rc!=SQLITE_OK ){ rc = TCL_ERROR; Tcl_SetResult(interp, (char *)sqlite3_errmsg(pDb->db), TCL_VOLATILE); } break; } |
︙ | ︙ |