/ Check-in [e0689f05d1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Initial implementation of user-defined Tcl aggregate functions. Testing and documentation to come.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | andygoth-tcl-function
Files: files | file ages | folders
SHA3-256: e0689f05d1f8792d9024d8f5a8dbd2a9dbe3b7d54ba4ea9e3f249c2a62f87fe2
User & Date: andygoth 2019-01-30 17:30:58
Context
2019-01-31
05:25
Add method argument to aggregate function invocations check-in: 4fc35d5e09 user: andygoth tags: andygoth-tcl-function
2019-01-30
17:30
Initial implementation of user-defined Tcl aggregate functions. Testing and documentation to come. check-in: e0689f05d1 user: andygoth tags: andygoth-tcl-function
14:01
Enhancements to the index_usage utility program. check-in: 19c739b4a8 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/tclsqlite.c.

895
896
897
898
899
900
901
902



903





904

905




906
907
908
909
910
911
912
913
914
915
916
917
918
  Tcl_ListObjAppendElement(p->interp, pCmd, Tcl_NewStringObj(zB, nB));
  Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT);
  Tcl_DecrRefCount(pCmd);
  return (atoi(Tcl_GetStringResult(p->interp)));
}

/*
** This routine is called to evaluate an SQL function implemented



** using TCL script.





*/

static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){




  SqlFunc *p = sqlite3_user_data(context);
  Tcl_Obj *pCmd;
  int i;
  int rc;

  if( argc==0 ){
    /* If there are no arguments to the function, call Tcl_EvalObjEx on the
    ** script object directly.  This allows the TCL compiler to generate
    ** bytecode for the command on the first invocation and thus make
    ** subsequent invocations much faster. */
    pCmd = p->pScript;
    Tcl_IncrRefCount(pCmd);
    rc = Tcl_EvalObjEx(p->interp, pCmd, 0);







|
>
>
>
|
>
>
>
>
>

>
|
>
>
>
>





|







895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
  Tcl_ListObjAppendElement(p->interp, pCmd, Tcl_NewStringObj(zB, nB));
  Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT);
  Tcl_DecrRefCount(pCmd);
  return (atoi(Tcl_GetStringResult(p->interp)));
}

/*
** Utility function shared by the following functions:
** - tclSqlFuncScalar()
** - tclSqlFuncStep()
** - tclSqlFuncFinal()
** This function calls a user-defined TCL script to implement an SQL function.
** Before being invoked, additional arguments may be appended to the script.
** Optionally, the first argument may be a TCL object.  All remaining arguments
** are TCL objects constructed from SQL values.  On success, the return value
** is a pointer to the TCL object result.  On failure, the return value is NULL,
** and an error message is stored in the SQL context.
*/
static Tcl_Obj *tclSqlFuncCall(
  sqlite3_context *context,
  Tcl_Obj *pState,
  int argc,
  sqlite3_value **argv
){
  SqlFunc *p = sqlite3_user_data(context);
  Tcl_Obj *pCmd;
  int i;
  int rc;

  if( argc==0 && !pState ){
    /* If there are no arguments to the function, call Tcl_EvalObjEx on the
    ** script object directly.  This allows the TCL compiler to generate
    ** bytecode for the command on the first invocation and thus make
    ** subsequent invocations much faster. */
    pCmd = p->pScript;
    Tcl_IncrRefCount(pCmd);
    rc = Tcl_EvalObjEx(p->interp, pCmd, 0);
927
928
929
930
931
932
933
934
935
936
937








938
939
940
941
942
943
944
    ** of the list to tclCmdNameType, that alternate representation will
    ** be preserved and reused on the next invocation.
    */
    Tcl_Obj **aArg;
    int nArg;
    if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){
      sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
      return;
    }
    pCmd = Tcl_NewListObj(nArg, aArg);
    Tcl_IncrRefCount(pCmd);








    for(i=0; i<argc; i++){
      sqlite3_value *pIn = argv[i];
      Tcl_Obj *pVal;

      /* Set pVal to contain the i'th column of this row. */
      switch( sqlite3_value_type(pIn) ){
        case SQLITE_BLOB: {







|



>
>
>
>
>
>
>
>







940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
    ** of the list to tclCmdNameType, that alternate representation will
    ** be preserved and reused on the next invocation.
    */
    Tcl_Obj **aArg;
    int nArg;
    if( Tcl_ListObjGetElements(p->interp, p->pScript, &nArg, &aArg) ){
      sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
      return 0;
    }
    pCmd = Tcl_NewListObj(nArg, aArg);
    Tcl_IncrRefCount(pCmd);
    if( pState ){
      rc = Tcl_ListObjAppendElement(p->interp, pCmd, pState);
      if( rc ){
        Tcl_DecrRefCount(pCmd);
        sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
        return 0;
      }
    }
    for(i=0; i<argc; i++){
      sqlite3_value *pIn = argv[i];
      Tcl_Obj *pVal;

      /* Set pVal to contain the i'th column of this row. */
      switch( sqlite3_value_type(pIn) ){
        case SQLITE_BLOB: {
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991

992
993















994
995



996
997
998
999
1000
1001
1002
1003
1004
          break;
        }
      }
      rc = Tcl_ListObjAppendElement(p->interp, pCmd, pVal);
      if( rc ){
        Tcl_DecrRefCount(pCmd);
        sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
        return;
      }
    }
    if( !p->useEvalObjv ){
      /* Tcl_EvalObjEx() will automatically call Tcl_EvalObjv() if pCmd
      ** is a list without a string representation.  To prevent this from
      ** happening, make sure pCmd has a valid string representation */
      Tcl_GetString(pCmd);
    }
    rc = Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT);
    Tcl_DecrRefCount(pCmd);
  }

  if( rc && rc!=TCL_RETURN ){
    sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);

  }else{
    Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);















    int n;
    u8 *data;



    const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
    char c = zType[0];
    if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
      /* Only return a BLOB type if the Tcl variable is a bytearray and
      ** has no string representation. */
      data = Tcl_GetByteArrayFromObj(pVar, &n);
      sqlite3_result_blob(context, data, n, SQLITE_TRANSIENT);
    }else if( c=='b' && strcmp(zType,"boolean")==0 ){
      Tcl_GetIntFromObj(0, pVar, &n);







|














>

|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
>
>
>
|
|







991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
          break;
        }
      }
      rc = Tcl_ListObjAppendElement(p->interp, pCmd, pVal);
      if( rc ){
        Tcl_DecrRefCount(pCmd);
        sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
        return 0;
      }
    }
    if( !p->useEvalObjv ){
      /* Tcl_EvalObjEx() will automatically call Tcl_EvalObjv() if pCmd
      ** is a list without a string representation.  To prevent this from
      ** happening, make sure pCmd has a valid string representation */
      Tcl_GetString(pCmd);
    }
    rc = Tcl_EvalObjEx(p->interp, pCmd, TCL_EVAL_DIRECT);
    Tcl_DecrRefCount(pCmd);
  }

  if( rc && rc!=TCL_RETURN ){
    sqlite3_result_error(context, Tcl_GetStringResult(p->interp), -1);
    return 0;
  }else{
    return Tcl_GetObjResult(p->interp);
  }
}

/*
** Utility function shared by the following functions:
** - tclSqlFuncScalar()
** - tclSqlFuncFinal()
** This function converts a TCL object result to an SQL value and stores it into
** the SQL context.  If the TCL object result pointer is NULL, i.e. if the TCL
** script result code was not TCL_OK or TCL_RETURN, no action is taken.
*/
static void tclSqlFuncResult(
  sqlite3_context *context,
  Tcl_Obj *pVar
){
  int n;
  u8 *data;
  const char *zType;
  char c;
  if( pVar ){
    zType = pVar->typePtr ? pVar->typePtr->name : "";
    c = zType[0];
    if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
      /* Only return a BLOB type if the Tcl variable is a bytearray and
      ** has no string representation. */
      data = Tcl_GetByteArrayFromObj(pVar, &n);
      sqlite3_result_blob(context, data, n, SQLITE_TRANSIENT);
    }else if( c=='b' && strcmp(zType,"boolean")==0 ){
      Tcl_GetIntFromObj(0, pVar, &n);
1014
1015
1016
1017
1018
1019
1020









































































































































1021
1022
1023
1024
1025
1026
1027
      sqlite3_result_int64(context, v);
    }else{
      data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
      sqlite3_result_text(context, (char *)data, n, SQLITE_TRANSIENT);
    }
  }
}










































































































































#ifndef SQLITE_OMIT_AUTHORIZATION
/*
** This is the authentication function.  It appends the authentication
** type code and the two arguments to zCmd[] then invokes the result
** on the interpreter.  The reply is examined to determine if the
** authentication fails or succeeds.







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
      sqlite3_result_int64(context, v);
    }else{
      data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
      sqlite3_result_text(context, (char *)data, n, SQLITE_TRANSIENT);
    }
  }
}

/*
** This address is distinct from all possible Tcl_Obj pointers.  It is used to
** signify that an error occurred in tclSqlFuncStep() and that tclSqlFuncFinal()
** must not invoke the TCL script again.
*/
static Tcl_Obj tclSqlFuncError;

/*
** Utility function shared by the following functions:
** - tclSqlFuncStep()
** - tclSqlFuncFinal()
** This function returns the address of a buffer consisting of a TCL object
** pointer.  The caller is expected to modify this to point to the TCL object
** that will be used as the first argument to the next invocation of the TCL
** script implementing an SQL aggregate function.
**
** Implementation note: When called from tclSqlFuncFinal(), it would be slightly
** faster to pass 0 rather than sizeof(*ppState) as the second argument to
** sqlite3_aggregate_context().  This would allow the no-rows case to avoid
** allocating a buffer that will be deallocated as soon as tclSqlFuncFinal()
** returns.  However, the added code complexity is not justified for a tiny bump
** in performance that would only be realized in a corner case, and it's already
** necessary to allocate an empty Tcl_Obj in this case, so be straightforward
** rather than clever.
*/
static Tcl_Obj **tclSqlFuncState(sqlite3_context *context){
  Tcl_Obj **ppState = sqlite3_aggregate_context(context, sizeof(*ppState));
  if( !*ppState ){
    *ppState = Tcl_NewObj();
    Tcl_IncrRefCount(*ppState);
  }
  return ppState;
}

/*
** This routine is called to evaluate a scalar SQL function implemented
** using TCL script.
*/
static void tclSqlFuncScalar(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  tclSqlFuncResult(context, tclSqlFuncCall(context, NULL, argc, argv));
}

/*
** This routine is called to evaluate an aggregate SQL function implemented
** using TCL script.  It is invoked for each row in the group.  The TCL result
** of each invocation becomes the first argument to the next invocation.  The
** first argument of the first invocation is empty string.
*/
static void tclSqlFuncStep(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){
  Tcl_Obj **ppState;
  Tcl_Obj *pResult;

  /* Even though this check was performed when the function was registered, it
  ** is necessary to double-check here in case -argcount was not used, meaning
  ** SQLite will allow the function to have any number of arguments, even zero.
  ** The presence or absence of arguments is how the TCL script can distinguish
  ** between being called as a step function or a final function.  Thus, being
  ** passed zero arguments would create ambiguity. */
  if( argc==0 ){
    sqlite3_result_error(context, "insufficient arguments: all user-defined "
        "TCL aggregate functions must have at least one argument", -1);
    return;
  }

  ppState = tclSqlFuncState(context);
  pResult = tclSqlFuncCall(context, *ppState, argc, argv);
  if( pResult ){
    /* If the step function succeeded, increment the new state reference count
    ** before decrementing the old state reference count.  This sequence will
    ** deallocate the old state if it is a distinct TCL object from the new, but
    ** it will not deallocate the old state if the TCL script modified it in
    ** place and then returned it again.
    **
    ** BUG: The above is not entirely true.  The reference count of *ppState is
    ** nonzero when it is passed to Tcl_EvalObjEx(), which further increments
    ** the reference count because it places it in an argument variable.  As a
    ** result, the TCL script sees a shared Tcl_Obj which cannot be modified in
    ** place.  Better performance could be had if the reference count were zero
    ** when passed to Tcl_EvalObjEx(), but this is dangerous because the Tcl_Obj
    ** will have zero reference count despite being pointed to by *ppState and
    ** is therefore theoretically vulnerable to surprise deallocation.
    **
    ** It is tempting to remove Tcl_IncrRefCount() from tclSqlFuncState() so
    ** that the Tcl_Obj have zero reference count, gambling that no uncontrolled
    ** code will be able to find and destroy the Tcl_Obj since the only pointer
    ** to it is here within the SQLite TCL binding.  However, not even this
    ** solves the problem, at least not in TCL 8.7, since the original argument
    ** value is still preserved inside the call stack diagnostics and therefore
    ** cannot be modified in place.
    **
    ** One way to fix this is to instead store the state data in a variable
    ** rather than pass it as a value, then pass the name of the variable to the
    ** script.  This will complicate the scripts, increasing maintenance burden
    ** and incurring further performance penalties which may offset any supposed
    ** performance gains arising from avoiding copying.
    **
    ** Another way to fix this is for SQLite to create custom TCL commands to
    ** read and write the value of the state Tcl_Obj.  But again, this will make
    ** the scripts more complicated, with the same costs as above.
    **
    ** The bottom line is that this bug is probably not worth fixing here, and
    ** it's up to TCL to find a better way to manage the lifetime of Tcl_Objs
    ** being held hostage by introspection systems that are not being used. */
    Tcl_IncrRefCount(pResult);
    Tcl_DecrRefCount(*ppState);
    *ppState = pResult;
  }else{
    /* If the step function had an error result, deallocate the TCL state object
    ** now and point to tclSqlFuncError so that tclSqlFuncFinal() will know not
    ** to invoke the TCL script again. */
    Tcl_DecrRefCount(*ppState);
    *ppState = &tclSqlFuncError;
  }
}

/*
** This routine is called to evaluate an aggregate SQL function implemented
** using TCL script.  It is invoked at the end of each group.  The TCL result
** of the prior invocation (via tclSqlFuncStep()) is the first argument to this
** invocation, or empty string if there were no rows.
*/
static void tclSqlFuncFinal(sqlite3_context *context){
  Tcl_Obj **ppState = tclSqlFuncState(context);
  if( *ppState!=&tclSqlFuncError ){
    tclSqlFuncResult(context, tclSqlFuncCall(context, *ppState, 0, 0));
    Tcl_DecrRefCount(*ppState);
  }
}

#ifndef SQLITE_OMIT_AUTHORIZATION
/*
** This is the authentication function.  It appends the authentication
** type code and the two arguments to zCmd[] then invokes the result
** on the interpreter.  The reply is examined to determine if the
** authentication fails or succeeds.
2636
2637
2638
2639
2640
2641
2642

2643
2644
2645
2646
2647
2648
2649
  /*
  **     $db function NAME [-argcount N] [-deterministic] SCRIPT
  **
  ** Create a new SQL function called NAME.  Whenever that function is
  ** called, invoke SCRIPT to evaluate the function.
  */
  case DB_FUNCTION: {

    int flags = SQLITE_UTF8;
    SqlFunc *pFunc;
    Tcl_Obj *pScript;
    char *zName;
    int nArg = -1;
    int i;
    if( objc<4 ){







>







2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
  /*
  **     $db function NAME [-argcount N] [-deterministic] SCRIPT
  **
  ** Create a new SQL function called NAME.  Whenever that function is
  ** called, invoke SCRIPT to evaluate the function.
  */
  case DB_FUNCTION: {
    enum {SCALAR, AGGREGATE} type = SCALAR;
    int flags = SQLITE_UTF8;
    SqlFunc *pFunc;
    Tcl_Obj *pScript;
    char *zName;
    int nArg = -1;
    int i;
    if( objc<4 ){
2664
2665
2666
2667
2668
2669
2670



2671
2672
2673
2674
2675
2676





2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688

2689
2690




2691
2692
2693
2694
2695
2696
2697
                           (char*)0);
          return TCL_ERROR;
        }
        i++;
      }else
      if( n>2 && strncmp(z, "-deterministic",n)==0 ){
        flags |= SQLITE_DETERMINISTIC;



      }else{
        Tcl_AppendResult(interp, "bad option \"", z,
            "\": must be -argcount or -deterministic", (char*)0
        );
        return TCL_ERROR;
      }





    }

    pScript = objv[objc-1];
    zName = Tcl_GetStringFromObj(objv[2], 0);
    pFunc = findSqlFunc(pDb, zName);
    if( pFunc==0 ) return TCL_ERROR;
    if( pFunc->pScript ){
      Tcl_DecrRefCount(pFunc->pScript);
    }
    pFunc->pScript = pScript;
    Tcl_IncrRefCount(pScript);
    pFunc->useEvalObjv = safeToUseEvalObjv(interp, pScript);

    rc = sqlite3_create_function(pDb->db, zName, nArg, flags,
        pFunc, tclSqlFunc, 0, 0);




    if( rc!=SQLITE_OK ){
      rc = TCL_ERROR;
      Tcl_SetResult(interp, (char *)sqlite3_errmsg(pDb->db), TCL_VOLATILE);
    }
    break;
  }








>
>
>


|



>
>
>
>
>












>
|
|
>
>
>
>







2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
                           (char*)0);
          return TCL_ERROR;
        }
        i++;
      }else
      if( n>2 && strncmp(z, "-deterministic",n)==0 ){
        flags |= SQLITE_DETERMINISTIC;
      }else
      if( n>2 && strncmp(z, "-aggregate",n)==0 ){
        type = AGGREGATE;
      }else{
        Tcl_AppendResult(interp, "bad option \"", z,
            "\": must be -argcount, -deterministic, or -aggregate", (char *)0
        );
        return TCL_ERROR;
      }
    }
    if( type==AGGREGATE && nArg==0 ){
      Tcl_AppendResult(interp, "all user-defined TCL aggregate functions must "
          "have at least one argument", (char *)0);
      return TCL_ERROR;
    }

    pScript = objv[objc-1];
    zName = Tcl_GetStringFromObj(objv[2], 0);
    pFunc = findSqlFunc(pDb, zName);
    if( pFunc==0 ) return TCL_ERROR;
    if( pFunc->pScript ){
      Tcl_DecrRefCount(pFunc->pScript);
    }
    pFunc->pScript = pScript;
    Tcl_IncrRefCount(pScript);
    pFunc->useEvalObjv = safeToUseEvalObjv(interp, pScript);
    if( type==SCALAR ){
      rc = sqlite3_create_function(pDb->db, zName, nArg, flags,
          pFunc, tclSqlFuncScalar, 0, 0);
    }else{
      rc = sqlite3_create_function(pDb->db, zName, nArg, flags,
          pFunc, 0, tclSqlFuncStep, tclSqlFuncFinal);
    }
    if( rc!=SQLITE_OK ){
      rc = TCL_ERROR;
      Tcl_SetResult(interp, (char *)sqlite3_errmsg(pDb->db), TCL_VOLATILE);
    }
    break;
  }