Fossil

Check-in [791a513c28]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Improvements to clone performance. Skip cross-linking during the clone the automatically rebuild after the clone. Fixes to rebuild to make sure all artifacts are crosslinked.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 791a513c289ded077964c0c1332e089a0e65587d
User & Date: drh 2008-05-18 17:18:41.000
Context
2008-05-18
17:29
Sync server responds to reqconfig cards. Allow cloning of repositories that contain tickets. check-in: ba8af8731f user: drh tags: trunk
17:18
Improvements to clone performance. Skip cross-linking during the clone the automatically rebuild after the clone. Fixes to rebuild to make sure all artifacts are crosslinked. check-in: 791a513c28 user: drh tags: trunk
15:51
Improvements to cloning performance. check-in: 243e02bfbd user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/db.c.
48
49
50
51
52
53
54

55
56
57
58
59
60
61
/*
** An single SQL statement is represented as an instance of the following
** structure.
*/
struct Stmt {
  Blob sql;               /* The SQL for this statement */
  sqlite3_stmt *pStmt;    /* The results of sqlite3_prepare() */

};
#endif /* INTERFACE */

/*
** Call this routine when a database error occurs.
*/
static void db_err(const char *zFormat, ...){







>







48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
/*
** An single SQL statement is represented as an instance of the following
** structure.
*/
struct Stmt {
  Blob sql;               /* The SQL for this statement */
  sqlite3_stmt *pStmt;    /* The results of sqlite3_prepare() */
  Stmt *pNext, *pPrev;    /* List of all unfinalized statements */
};
#endif /* INTERFACE */

/*
** Call this routine when a database error occurs.
*/
static void db_err(const char *zFormat, ...){
80
81
82
83
84
85
86

87
88
89
90
91
92
93
static int isNewRepo = 0;   /* True if the repository is newly created */
static int doRollback = 0;  /* True to force a rollback */
static int nCommitHook = 0; /* Number of commit hooks */
static struct sCommitHook {
  int (*xHook)(void);  /* Functions to call at db_end_transaction() */
  int sequence;        /* Call functions in sequence order */
} aHook[5];


/*
** This routine is called by the SQLite commit-hook mechanism
** just prior to each omit.  All this routine does is verify
** that nBegin really is zero.  That insures that transactions
** cannot commit by any means other than by calling db_end_transaction()
** below.







>







81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
static int isNewRepo = 0;   /* True if the repository is newly created */
static int doRollback = 0;  /* True to force a rollback */
static int nCommitHook = 0; /* Number of commit hooks */
static struct sCommitHook {
  int (*xHook)(void);  /* Functions to call at db_end_transaction() */
  int sequence;        /* Call functions in sequence order */
} aHook[5];
static Stmt *pAllStmt = 0;  /* List of all unfinalized statements */

/*
** This routine is called by the SQLite commit-hook mechanism
** just prior to each omit.  All this routine does is verify
** that nBegin really is zero.  That insures that transactions
** cannot commit by any means other than by calling db_end_transaction()
** below.
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188

189
190
191

192


193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209




210
211
212
213
214
215
216
    }
  }
  aHook[nCommitHook].sequence = sequence;
  aHook[nCommitHook].xHook = x;
  nCommitHook++;
}

/*
** Prepare or reprepare the sqlite3 statement from the raw SQL text.
*/
static void reprepare(Stmt *pStmt){
  sqlite3_stmt *pNew;
  if( sqlite3_prepare(g.db, blob_buffer(&pStmt->sql), -1, &pNew, 0)!=0 ){
    db_err("%s\n%s", blob_str(&pStmt->sql), sqlite3_errmsg(g.db));
  }
  if( pStmt->pStmt ){
    sqlite3_transfer_bindings(pStmt->pStmt, pNew);
    sqlite3_finalize(pStmt->pStmt);
  }
  pStmt->pStmt = pNew;
}

/*
** Prepare a Stmt.  Assume that the Stmt is previously uninitialized.
** If the input string contains multiple SQL statements, only the first
** one is processed.  All statements beyond the first are silently ignored.
*/
int db_vprepare(Stmt *pStmt, const char *zFormat, va_list ap){

  blob_zero(&pStmt->sql);
  blob_vappendf(&pStmt->sql, zFormat, ap);
  va_end(ap);

  pStmt->pStmt = 0;


  reprepare(pStmt);
  return 0;
}
int db_prepare(Stmt *pStmt, const char *zFormat, ...){
  int rc;
  va_list ap;
  va_start(ap, zFormat);
  rc = db_vprepare(pStmt, zFormat, ap);
  va_end(ap);
  return rc;
}
int db_static_prepare(Stmt *pStmt, const char *zFormat, ...){
  int rc = SQLITE_OK;
  if( blob_size(&pStmt->sql)==0 ){
    va_list ap;
    va_start(ap, zFormat);
    rc = db_vprepare(pStmt, zFormat, ap);




    va_end(ap);
  }
  return rc;
}

/*
** Return the index of a bind parameter







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<






>



>
|
>
>
|
















>
>
>
>







163
164
165
166
167
168
169















170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
    }
  }
  aHook[nCommitHook].sequence = sequence;
  aHook[nCommitHook].xHook = x;
  nCommitHook++;
}
















/*
** Prepare a Stmt.  Assume that the Stmt is previously uninitialized.
** If the input string contains multiple SQL statements, only the first
** one is processed.  All statements beyond the first are silently ignored.
*/
int db_vprepare(Stmt *pStmt, const char *zFormat, va_list ap){
  char *zSql;
  blob_zero(&pStmt->sql);
  blob_vappendf(&pStmt->sql, zFormat, ap);
  va_end(ap);
  zSql = blob_str(&pStmt->sql);
  if( sqlite3_prepare_v2(g.db, zSql, -1, &pStmt->pStmt, 0)!=0 ){
    db_err("%s\n%s", zSql, sqlite3_errmsg(g.db));
  }
  pStmt->pNext = pStmt->pPrev = 0;
  return 0;
}
int db_prepare(Stmt *pStmt, const char *zFormat, ...){
  int rc;
  va_list ap;
  va_start(ap, zFormat);
  rc = db_vprepare(pStmt, zFormat, ap);
  va_end(ap);
  return rc;
}
int db_static_prepare(Stmt *pStmt, const char *zFormat, ...){
  int rc = SQLITE_OK;
  if( blob_size(&pStmt->sql)==0 ){
    va_list ap;
    va_start(ap, zFormat);
    rc = db_vprepare(pStmt, zFormat, ap);
    pStmt->pNext = pAllStmt;
    pStmt->pPrev = 0;
    if( pAllStmt ) pAllStmt->pPrev = pStmt;
    pAllStmt = pStmt;
    va_end(ap);
  }
  return rc;
}

/*
** Return the index of a bind parameter
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291











292
293
294
295
296
297
298
}

/*
** Step the SQL statement.  Return either SQLITE_ROW or an error code
** or SQLITE_OK if the statement finishes successfully.
*/
int db_step(Stmt *pStmt){
  int rc = SQLITE_OK;
  int limit = 3;
  while( limit-- ){
    rc = sqlite3_step(pStmt->pStmt);
    if( rc==SQLITE_ERROR ){
      rc = sqlite3_reset(pStmt->pStmt);
    }
    if( rc==SQLITE_SCHEMA ){
      reprepare(pStmt);
    }else{
      break;
    }
  }
  return rc;
}

/*
** Reset or finalize a statement.
*/
int db_reset(Stmt *pStmt){
  int rc = sqlite3_reset(pStmt->pStmt);
  db_check_result(rc);
  return rc;
}
int db_finalize(Stmt *pStmt){
  int rc;
  blob_reset(&pStmt->sql);
  rc = sqlite3_finalize(pStmt->pStmt);
  db_check_result(rc);











  return rc;
}

/*
** Return the rowid of the most recent insert
*/
i64 db_last_insert_rowid(void){







|
<
<
|
<
<
<
<
<
<
<
<
<
















>
>
>
>
>
>
>
>
>
>
>







251
252
253
254
255
256
257
258


259









260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
}

/*
** Step the SQL statement.  Return either SQLITE_ROW or an error code
** or SQLITE_OK if the statement finishes successfully.
*/
int db_step(Stmt *pStmt){
  int rc;


  rc = sqlite3_step(pStmt->pStmt);









  return rc;
}

/*
** Reset or finalize a statement.
*/
int db_reset(Stmt *pStmt){
  int rc = sqlite3_reset(pStmt->pStmt);
  db_check_result(rc);
  return rc;
}
int db_finalize(Stmt *pStmt){
  int rc;
  blob_reset(&pStmt->sql);
  rc = sqlite3_finalize(pStmt->pStmt);
  db_check_result(rc);
  pStmt->pStmt = 0;
  if( pStmt->pNext ){
    pStmt->pNext->pPrev = pStmt->pPrev;
  }
  if( pStmt->pPrev ){
    pStmt->pPrev->pNext = pStmt->pNext;
  }else if( pAllStmt==pStmt ){
    pAllStmt = pStmt->pNext;
  }
  pStmt->pNext = 0;
  pStmt->pPrev = 0;
  return rc;
}

/*
** Return the rowid of the most recent insert
*/
i64 db_last_insert_rowid(void){
712
713
714
715
716
717
718



719
720
721
722
723
724
725
}

/*
** Close the database connection.
*/
void db_close(void){
  if( g.db==0 ) return;



  g.repositoryOpen = 0;
  g.localOpen = 0;
  g.configOpen = 0;
  sqlite3_close(g.db);
  g.db = 0;
}








>
>
>







707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
}

/*
** Close the database connection.
*/
void db_close(void){
  if( g.db==0 ) return;
  while( pAllStmt ){
    db_finalize(pAllStmt);
  }
  g.repositoryOpen = 0;
  g.localOpen = 0;
  g.configOpen = 0;
  sqlite3_close(g.db);
  g.db = 0;
}

Changes to src/rebuild.c.
66
67
68
69
70
71
72

73
74
75
76
77


78
79
80
81
82
83
84

/*
** Variables used for progress information
*/
static int totalSize;       /* Total number of artifacts to process */
static int processCnt;      /* Number processed so far */
static int ttyOutput;       /* Do progress output */


/*
** Called after each artifact is processed
*/
static void rebuild_step_done(void){


  if( ttyOutput ){
    processCnt++;
    printf("%d (%d%%)...\r", processCnt, (processCnt*100/totalSize));
    fflush(stdout);
  }
}








>




|
>
>







66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

/*
** Variables used for progress information
*/
static int totalSize;       /* Total number of artifacts to process */
static int processCnt;      /* Number processed so far */
static int ttyOutput;       /* Do progress output */
static Bag bagDone;         /* Bag of records rebuilt */

/*
** Called after each artifact is processed
*/
static void rebuild_step_done(rid){
  assert( bag_find(&bagDone, rid)==0 );
  bag_insert(&bagDone, rid);
  if( ttyOutput ){
    processCnt++;
    printf("%d (%d%%)...\r", processCnt, (processCnt*100/totalSize));
    fflush(stdout);
  }
}

101
102
103
104
105
106
107


108

109
110
111
112
113
114
115
    );
  }

  /* Find all children of artifact rid */
  db_prepare(&q1, "SELECT rid FROM delta WHERE srcid=%d", rid);
  bag_init(&children);
  while( db_step(&q1)==SQLITE_ROW ){


    bag_insert(&children, db_column_int(&q1, 0));

  }
  nChild = bag_count(&children);
  db_finalize(&q1);

  /* Crosslink the artifact */
  if( nChild==0 ){
    pUse = pBase;







>
>
|
>







104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    );
  }

  /* Find all children of artifact rid */
  db_prepare(&q1, "SELECT rid FROM delta WHERE srcid=%d", rid);
  bag_init(&children);
  while( db_step(&q1)==SQLITE_ROW ){
    int cid = db_column_int(&q1, 0);
    if( !bag_find(&bagDone, cid) ){
      bag_insert(&children, cid);
    }
  }
  nChild = bag_count(&children);
  db_finalize(&q1);

  /* Crosslink the artifact */
  if( nChild==0 ){
    pUse = pBase;
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167

168
169
170
171
172
173
174
      rebuild_step(cid, sz, pUse);
    }else{
      db_finalize(&q2);
      blob_reset(pUse);
    }
  }
  bag_clear(&children);
  rebuild_step_done();
}

/*
** Core function to rebuild the infomration in the derived tables of a
** fossil repository from the blobs. This function is shared between
** 'rebuild_database' ('rebuild') and 'reconstruct_cmd'
** ('reconstruct'), both of which have to regenerate this information
** from scratch.
**
** If the randomize parameter is true, then the BLOBs are deliberately
** extracted in a random order.  This feature is used to test the
** ability of fossil to accept records in any order and still
** construct a sane repository.
*/
int rebuild_db(int randomize, int doOut){
  Stmt s;
  int errCnt = 0;
  char *zTable;


  ttyOutput = doOut;
  processCnt = 0;
  db_multi_exec(zSchemaUpdates);
  for(;;){
    zTable = db_text(0,
       "SELECT name FROM sqlite_master"
       " WHERE type='table'"







|



















>







147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
      rebuild_step(cid, sz, pUse);
    }else{
      db_finalize(&q2);
      blob_reset(pUse);
    }
  }
  bag_clear(&children);
  rebuild_step_done(rid);
}

/*
** Core function to rebuild the infomration in the derived tables of a
** fossil repository from the blobs. This function is shared between
** 'rebuild_database' ('rebuild') and 'reconstruct_cmd'
** ('reconstruct'), both of which have to regenerate this information
** from scratch.
**
** If the randomize parameter is true, then the BLOBs are deliberately
** extracted in a random order.  This feature is used to test the
** ability of fossil to accept records in any order and still
** construct a sane repository.
*/
int rebuild_db(int randomize, int doOut){
  Stmt s;
  int errCnt = 0;
  char *zTable;

  bag_init(&bagDone);
  ttyOutput = doOut;
  processCnt = 0;
  db_multi_exec(zSchemaUpdates);
  for(;;){
    zTable = db_text(0,
       "SELECT name FROM sqlite_master"
       " WHERE type='table'"
202
203
204
205
206
207
208
















209
210
211
212
213
214
215
216
217
218
  while( db_step(&s)==SQLITE_ROW ){
    int rid = db_column_int(&s, 0);
    int size = db_column_int(&s, 1);
    if( size>=0 ){
      Blob content;
      content_get(rid, &content);
      rebuild_step(rid, size, &content);
















    }else{
      db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
      rebuild_step_done();
    }
  }
  db_finalize(&s);
  if( ttyOutput ){
    printf("\n");
  }
  return errCnt;







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


|







209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
  while( db_step(&s)==SQLITE_ROW ){
    int rid = db_column_int(&s, 0);
    int size = db_column_int(&s, 1);
    if( size>=0 ){
      Blob content;
      content_get(rid, &content);
      rebuild_step(rid, size, &content);
    }
  }
  db_finalize(&s);
  db_prepare(&s,
     "SELECT rid, size FROM blob"
     " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
  );
  while( db_step(&s)==SQLITE_ROW ){
    int rid = db_column_int(&s, 0);
    int size = db_column_int(&s, 1);
    if( size>=0 ){
      if( !bag_find(&bagDone, rid) ){
        Blob content;
        content_get(rid, &content);
        rebuild_step(rid, size, &content);
      }
    }else{
      db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
      rebuild_step_done(rid);
    }
  }
  db_finalize(&s);
  if( ttyOutput ){
    printf("\n");
  }
  return errCnt;