Fossil

Check-in [b81112371c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the --sqlstats command-line option for use in optimization work. Use a persistent prepared statement for a single query to reduce the number of calls to sqlite3_prepare() for the "update" command.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: b81112371c25058de6bf225c58cfec5ec22ee1d4
User & Date: drh 2011-02-24 23:26:15.810
Context
2011-02-24
23:41
Immediately stop processing the "all" command if any individual operation fails, unless the --dontstop command-line option is used. check-in: f5a9e8e61e user: drh tags: trunk
23:26
Add the --sqlstats command-line option for use in optimization work. Use a persistent prepared statement for a single query to reduce the number of calls to sqlite3_prepare() for the "update" command. check-in: b81112371c user: drh tags: trunk
20:00
Reuse prepared statements to make clone run must faster on the server side. check-in: 1f41b0421d user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/content.c.
125
126
127
128
129
130
131















132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
    srcid = db_column_int(&q, 0);
  }else{
    srcid = 0;
  }
  db_reset(&q);
  return srcid;
}
















/*
** Check to see if content is available for artifact "rid".  Return
** true if it is.  Return false if rid is a phantom or depends on
** a phantom.
*/
int content_is_available(int rid){
  int srcid;
  int depth = 0;  /* Limit to recursion depth */
  while( depth++ < 10000000 ){  
    if( bag_find(&contentCache.missing, rid) ){
      return 0;
    }
    if( bag_find(&contentCache.available, rid) ){
      return 1;
    }
    if( db_int(-1, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
      bag_insert(&contentCache.missing, rid);
      return 0;
    }
    srcid = findSrcid(rid);
    if( srcid==0 ){
      bag_insert(&contentCache.available, rid);
      return 1;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
















|







125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
    srcid = db_column_int(&q, 0);
  }else{
    srcid = 0;
  }
  db_reset(&q);
  return srcid;
}

/*
** Return the blob.size field given blob.rid
*/
int content_size(int rid, int dflt){
  static Stmt q;
  int sz = dflt;
  db_static_prepare(&q, "SELECT size FROM blob WHERE rid=:r");
  db_bind_int(&q, ":r", rid);
  if( db_step(&q)==SQLITE_ROW ){
    sz = db_column_int(&q, 0);
  }
  db_reset(&q);
  return sz;
}

/*
** Check to see if content is available for artifact "rid".  Return
** true if it is.  Return false if rid is a phantom or depends on
** a phantom.
*/
int content_is_available(int rid){
  int srcid;
  int depth = 0;  /* Limit to recursion depth */
  while( depth++ < 10000000 ){  
    if( bag_find(&contentCache.missing, rid) ){
      return 0;
    }
    if( bag_find(&contentCache.available, rid) ){
      return 1;
    }
    if( content_size(rid, -1)<0 ){
      bag_insert(&contentCache.missing, rid);
      return 0;
    }
    srcid = findSrcid(rid);
    if( srcid==0 ){
      bag_insert(&contentCache.available, rid);
      return 1;
Changes to src/db.c.
86
87
88
89
90
91
92

93
94
95
96
97
98
99
static int doRollback = 0;  /* True to force a rollback */
static int nCommitHook = 0; /* Number of commit hooks */
static struct sCommitHook {
  int (*xHook)(void);  /* Functions to call at db_end_transaction() */
  int sequence;        /* Call functions in sequence order */
} aHook[5];
static Stmt *pAllStmt = 0;  /* List of all unfinalized statements */


/*
** This routine is called by the SQLite commit-hook mechanism
** just prior to each commit.  All this routine does is verify
** that nBegin really is zero.  That insures that transactions
** cannot commit by any means other than by calling db_end_transaction()
** below.







>







86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
static int doRollback = 0;  /* True to force a rollback */
static int nCommitHook = 0; /* Number of commit hooks */
static struct sCommitHook {
  int (*xHook)(void);  /* Functions to call at db_end_transaction() */
  int sequence;        /* Call functions in sequence order */
} aHook[5];
static Stmt *pAllStmt = 0;  /* List of all unfinalized statements */
static int nPrepare = 0;    /* Number of calls to sqlite3_prepare() */

/*
** This routine is called by the SQLite commit-hook mechanism
** just prior to each commit.  All this routine does is verify
** that nBegin really is zero.  That insures that transactions
** cannot commit by any means other than by calling db_end_transaction()
** below.
197
198
199
200
201
202
203

204
205
206
207
208
209
210
int db_vprepare(Stmt *pStmt, int errOk, const char *zFormat, va_list ap){
  int rc;
  char *zSql;
  blob_zero(&pStmt->sql);
  blob_vappendf(&pStmt->sql, zFormat, ap);
  va_end(ap);
  zSql = blob_str(&pStmt->sql);

  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt->pStmt, 0);
  if( rc!=0 && !errOk ){
    db_err("%s\n%s", sqlite3_errmsg(g.db), zSql);
  }
  pStmt->pNext = pStmt->pPrev = 0;
  pStmt->nStep = 0;
  return rc;







>







198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
int db_vprepare(Stmt *pStmt, int errOk, const char *zFormat, va_list ap){
  int rc;
  char *zSql;
  blob_zero(&pStmt->sql);
  blob_vappendf(&pStmt->sql, zFormat, ap);
  va_end(ap);
  zSql = blob_str(&pStmt->sql);
  nPrepare++;
  rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt->pStmt, 0);
  if( rc!=0 && !errOk ){
    db_err("%s\n%s", sqlite3_errmsg(g.db), zSql);
  }
  pStmt->pNext = pStmt->pPrev = 0;
  pStmt->nStep = 0;
  return rc;
734
735
736
737
738
739
740

741
742
743
744
745
746
747
  db_open_repository(0);

  /* If the "isexe" column is missing from the vfile table, then
  ** add it now.   This code added on 2010-03-06.  After all users have
  ** upgraded, this code can be safely deleted. 
  */
  rc = sqlite3_prepare(g.db, "SELECT isexe FROM vfile", -1, &pStmt, 0);

  sqlite3_finalize(pStmt);
  if( rc==SQLITE_ERROR ){
    sqlite3_exec(g.db, "ALTER TABLE vfile ADD COLUMN isexe BOOLEAN", 0, 0, 0);
  }

#if 0
  /* If the "mtime" column is missing from the vfile table, then







>







736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
  db_open_repository(0);

  /* If the "isexe" column is missing from the vfile table, then
  ** add it now.   This code added on 2010-03-06.  After all users have
  ** upgraded, this code can be safely deleted. 
  */
  rc = sqlite3_prepare(g.db, "SELECT isexe FROM vfile", -1, &pStmt, 0);
  nPrepare++;
  sqlite3_finalize(pStmt);
  if( rc==SQLITE_ERROR ){
    sqlite3_exec(g.db, "ALTER TABLE vfile ADD COLUMN isexe BOOLEAN", 0, 0, 0);
  }

#if 0
  /* If the "mtime" column is missing from the vfile table, then
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994

995
996
997
998
999
1000
1001
**
** Check for unfinalized statements and report errors if the reportErrors
** argument is true.  Ignore unfinalized statements when false.
*/
void db_close(int reportErrors){
  sqlite3_stmt *pStmt;
  if( g.db==0 ) return;
  if( g.fSqlTrace ){
    int cur, hiwtr;
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- LOOKASIDE_USED %10d %10d\n", cur, hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &cur, &hiwtr, 0);
    fprintf(stderr, "-- LOOKASIDE_HIT             %10d\n", hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &cur,&hiwtr,0);
    fprintf(stderr, "-- LOOKASIDE_MISS_SIZE       %10d\n", hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &cur,&hiwtr,0);
    fprintf(stderr, "-- LOOKASIDE_MISS_FULL       %10d\n", hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_CACHE_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- CACHE_USED     %10d\n", cur);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_SCHEMA_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- SCHEMA_USED    %10d\n", cur);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_STMT_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- STMT_USED      %10d\n", cur);
    sqlite3_status(SQLITE_STATUS_MEMORY_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- MEMORY_USED    %10d %10d\n", cur, hiwtr);
    sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &cur, &hiwtr, 0);
    fprintf(stderr, "-- MALLOC_SIZE               %10d\n", hiwtr);
    sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &cur, &hiwtr, 0);
    fprintf(stderr, "-- MALLOC_COUNT   %10d %10d\n", cur, hiwtr);
    sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &cur, &hiwtr, 0);
    fprintf(stderr, "-- PCACHE_OVFLOW  %10d %10d\n", cur, hiwtr);

  }
  while( pAllStmt ){
    db_finalize(pAllStmt);
  }
  db_end_transaction(1);
  pStmt = 0;
  if( reportErrors ){







|


|

|

|

|

|

|

|

|

|

|

|
>







967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
**
** Check for unfinalized statements and report errors if the reportErrors
** argument is true.  Ignore unfinalized statements when false.
*/
void db_close(int reportErrors){
  sqlite3_stmt *pStmt;
  if( g.db==0 ) return;
  if( g.fSqlStats ){
    int cur, hiwtr;
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- LOOKASIDE_USED         %10d %10d\n", cur, hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &cur, &hiwtr, 0);
    fprintf(stderr, "-- LOOKASIDE_HIT                     %10d\n", hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &cur,&hiwtr,0);
    fprintf(stderr, "-- LOOKASIDE_MISS_SIZE               %10d\n", hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &cur,&hiwtr,0);
    fprintf(stderr, "-- LOOKASIDE_MISS_FULL               %10d\n", hiwtr);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_CACHE_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- CACHE_USED             %10d\n", cur);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_SCHEMA_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- SCHEMA_USED            %10d\n", cur);
    sqlite3_db_status(g.db, SQLITE_DBSTATUS_STMT_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- STMT_USED              %10d\n", cur);
    sqlite3_status(SQLITE_STATUS_MEMORY_USED, &cur, &hiwtr, 0);
    fprintf(stderr, "-- MEMORY_USED            %10d %10d\n", cur, hiwtr);
    sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &cur, &hiwtr, 0);
    fprintf(stderr, "-- MALLOC_SIZE                       %10d\n", hiwtr);
    sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &cur, &hiwtr, 0);
    fprintf(stderr, "-- MALLOC_COUNT           %10d %10d\n", cur, hiwtr);
    sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &cur, &hiwtr, 0);
    fprintf(stderr, "-- PCACHE_OVFLOW          %10d %10d\n", cur, hiwtr);
    fprintf(stderr, "-- prepared statements    %10d\n", nPrepare);
  }
  while( pAllStmt ){
    db_finalize(pAllStmt);
  }
  db_end_transaction(1);
  pStmt = 0;
  if( reportErrors ){
Changes to src/main.c.
58
59
60
61
62
63
64
65

66
67
68
69
70
71
72
  int repositoryOpen;     /* True if the main repository database is open */
  char *zRepositoryName;  /* Name of the repository database */
  const char *zMainDbType;/* "configdb", "localdb", or "repository" */
  const char *zHome;      /* Name of user home directory */
  int localOpen;          /* True if the local database is open */
  char *zLocalRoot;       /* The directory holding the  local database */
  int minPrefix;          /* Number of digits needed for a distinct UUID */
  int fSqlTrace;          /* True if -sqltrace flag is present */

  int fSqlPrint;          /* True if -sqlprint flag is present */
  int fQuiet;             /* True if -quiet flag is present */
  int fHttpTrace;         /* Trace outbound HTTP requests */
  int fNoSync;            /* Do not do an autosync even.  --nosync */
  char *zPath;            /* Name of webpage being served */
  char *zExtra;           /* Extra path information past the webpage name */
  char *zBaseURL;         /* Full text of the URL being served */







|
>







58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
  int repositoryOpen;     /* True if the main repository database is open */
  char *zRepositoryName;  /* Name of the repository database */
  const char *zMainDbType;/* "configdb", "localdb", or "repository" */
  const char *zHome;      /* Name of user home directory */
  int localOpen;          /* True if the local database is open */
  char *zLocalRoot;       /* The directory holding the  local database */
  int minPrefix;          /* Number of digits needed for a distinct UUID */
  int fSqlTrace;          /* True if --sqltrace flag is present */
  int fSqlStats;          /* True if --sqltrace or --sqlstats are present */
  int fSqlPrint;          /* True if -sqlprint flag is present */
  int fQuiet;             /* True if -quiet flag is present */
  int fHttpTrace;         /* Trace outbound HTTP requests */
  int fNoSync;            /* Do not do an autosync even.  --nosync */
  char *zPath;            /* Name of webpage being served */
  char *zExtra;           /* Extra path information past the webpage name */
  char *zBaseURL;         /* Full text of the URL being served */
238
239
240
241
242
243
244


245
246
247
248
249
250
251
                    "\"%s help\" for a list of available commands\n"
                    "\"%s help COMMAND\" for specific details\n",
                    argv[0], argv[0], argv[0]);
    fossil_exit(1);
  }else{
    g.fQuiet = find_option("quiet", 0, 0)!=0;
    g.fSqlTrace = find_option("sqltrace", 0, 0)!=0;


    g.fSqlPrint = find_option("sqlprint", 0, 0)!=0;
    g.fHttpTrace = find_option("httptrace", 0, 0)!=0;
    g.zLogin = find_option("user", "U", 1);
    if( find_option("help",0,0)!=0 ){
      /* --help anywhere on the command line is translated into
      ** "fossil help argv[1] argv[2]..." */
      int i;







>
>







239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
                    "\"%s help\" for a list of available commands\n"
                    "\"%s help COMMAND\" for specific details\n",
                    argv[0], argv[0], argv[0]);
    fossil_exit(1);
  }else{
    g.fQuiet = find_option("quiet", 0, 0)!=0;
    g.fSqlTrace = find_option("sqltrace", 0, 0)!=0;
    g.fSqlStats = find_option("sqlstats", 0, 0)!=0;
    if( g.fSqlTrace ) g.fSqlStats = 1;
    g.fSqlPrint = find_option("sqlprint", 0, 0)!=0;
    g.fHttpTrace = find_option("httptrace", 0, 0)!=0;
    g.zLogin = find_option("user", "U", 1);
    if( find_option("help",0,0)!=0 ){
      /* --help anywhere on the command line is translated into
      ** "fossil help argv[1] argv[2]..." */
      int i;
Changes to src/verify.c.
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
** without error.
**
** Panic if anything goes wrong.  If this procedure returns it means
** that everything is OK.
*/
static void verify_rid(int rid){
  Blob uuid, hash, content;
  if( db_int(0, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
    return;  /* No way to verify phantoms */
  }
  blob_zero(&uuid);
  db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", rid);
  if( blob_size(&uuid)!=UUID_SIZE ){
    fossil_panic("not a valid rid: %d", rid);
  }







|







33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
** without error.
**
** Panic if anything goes wrong.  If this procedure returns it means
** that everything is OK.
*/
static void verify_rid(int rid){
  Blob uuid, hash, content;
  if( content_size(rid, 0)<0 ){
    return;  /* No way to verify phantoms */
  }
  blob_zero(&uuid);
  db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", rid);
  if( blob_size(&uuid)!=UUID_SIZE ){
    fossil_panic("not a valid rid: %d", rid);
  }
Changes to src/vfile.c.
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    "INSERT INTO vfile(vid,rid,mrid,pathname) "
    " VALUES(:vid,:id,:id,:name)");
  db_bind_int(&ins, ":vid", vid);
  manifest_file_rewind(p);
  while( (pFile = manifest_file_next(p,0))!=0 ){
    if( pFile->zUuid==0 || uuid_is_shunned(pFile->zUuid) ) continue;
    rid = uuid_to_rid(pFile->zUuid, 0);
    if( rid==0 || db_int(-1, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
      fossil_warning("content missing for %s", pFile->zName);
      continue;
    }
    db_bind_int(&ins, ":id", rid);
    db_bind_text(&ins, ":name", pFile->zName);
    db_step(&ins);
    db_reset(&ins);







|







90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
    "INSERT INTO vfile(vid,rid,mrid,pathname) "
    " VALUES(:vid,:id,:id,:name)");
  db_bind_int(&ins, ":vid", vid);
  manifest_file_rewind(p);
  while( (pFile = manifest_file_next(p,0))!=0 ){
    if( pFile->zUuid==0 || uuid_is_shunned(pFile->zUuid) ) continue;
    rid = uuid_to_rid(pFile->zUuid, 0);
    if( rid==0 || content_size(rid, -1)<0 ){
      fossil_warning("content missing for %s", pFile->zName);
      continue;
    }
    db_bind_int(&ins, ":id", rid);
    db_bind_text(&ins, ":name", pFile->zName);
    db_step(&ins);
    db_reset(&ins);