Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | The delta compress on xfer is working better now, but still needs work. |
|---|---|
| Downloads: | Tarball | ZIP archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
73bddaebb9ec5d6eea26b46f56d2b82b |
| User & Date: | drh 2007-08-09 17:42:59.000 |
Context
|
2007-08-09
| ||
| 19:07 | Additional work on the xfer mechanism, trying to increase the use of delta compression. ... (check-in: bd3c1d0023 user: drh tags: trunk) | |
| 17:42 | The delta compress on xfer is working better now, but still needs work. ... (check-in: 73bddaebb9 user: drh tags: trunk) | |
| 11:55 | Fix an off-by-one error that can cause a segfault during sync. ... (check-in: 6eca3132fe user: drh tags: trunk) | |
Changes
Changes to src/content.c.
| ︙ | ︙ | |||
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 |
db_prepare(&s1,
"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
g.rcvid, size, rid
);
blob_compress(pBlob, &cmpr);
db_bind_blob(&s1, ":data", &cmpr);
db_exec(&s1);
}else{
/* We are creating a new entry */
db_prepare(&s1,
"INSERT INTO blob(rcvid,size,uuid,content)"
"VALUES(%d,%d,'%s',:data)",
g.rcvid, size, blob_str(&hash)
);
if( pBlob ){
blob_compress(pBlob, &cmpr);
db_bind_blob(&s1, ":data", &cmpr);
}
db_exec(&s1);
rid = db_last_insert_rowid();
}
/* Finish the transaction and cleanup */
db_finalize(&s1);
db_end_transaction(0);
blob_reset(&hash);
| > > > > | 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 |
db_prepare(&s1,
"UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
g.rcvid, size, rid
);
blob_compress(pBlob, &cmpr);
db_bind_blob(&s1, ":data", &cmpr);
db_exec(&s1);
db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
}else{
/* We are creating a new entry */
db_prepare(&s1,
"INSERT INTO blob(rcvid,size,uuid,content)"
"VALUES(%d,%d,'%s',:data)",
g.rcvid, size, blob_str(&hash)
);
if( pBlob ){
blob_compress(pBlob, &cmpr);
db_bind_blob(&s1, ":data", &cmpr);
}
db_exec(&s1);
rid = db_last_insert_rowid();
if( !pBlob ){
db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
}
}
/* Finish the transaction and cleanup */
db_finalize(&s1);
db_end_transaction(0);
blob_reset(&hash);
|
| ︙ | ︙ |
Changes to src/delta.c.
| ︙ | ︙ | |||
331 332 333 334 335 336 337 |
landmark[hv] = i/NHASH;
}
/* Begin scanning the target file and generating copy commands and
** literal sections of the delta.
*/
base = 0; /* We have already generated everything before zOut[base] */
| | | 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 |
landmark[hv] = i/NHASH;
}
/* Begin scanning the target file and generating copy commands and
** literal sections of the delta.
*/
base = 0; /* We have already generated everything before zOut[base] */
while( base+NHASH<lenOut ){
int iSrc, iBlock;
unsigned int bestCnt, bestOfst, bestLitsz;
hash_init(&h, &zOut[base]);
i = 0; /* Trying to match a landmark against zOut[base+i] */
bestCnt = 0;
while( 1 ){
int hv;
|
| ︙ | ︙ |
Changes to src/rebuild.c.
| ︙ | ︙ | |||
58 59 60 61 62 63 64 |
" AND name NOT IN ('blob','delta','rcvfrom','user','config')");
if( zTable==0 ) break;
db_multi_exec("DROP TABLE %Q", zTable);
free(zTable);
}
db_multi_exec(zRepositorySchema2);
| | > > | | | | > > > | 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
" AND name NOT IN ('blob','delta','rcvfrom','user','config')");
if( zTable==0 ) break;
db_multi_exec("DROP TABLE %Q", zTable);
free(zTable);
}
db_multi_exec(zRepositorySchema2);
db_prepare(&s, "SELECT rid, size FROM blob");
while( db_step(&s)==SQLITE_ROW ){
int rid = db_column_int(&s, 0);
int size = db_column_int(&s, 1);
if( size>=0 ){
Blob content;
content_get(rid, &content);
manifest_crosslink(rid, &content);
blob_reset(&content);
}else{
db_multi_exec("INSERT INTO phantom VALUES(%d)", rid);
}
}
if( errCnt && !forceFlag ){
printf("%d errors. Rolling back changes. Use --force to force a commit.\n",
errCnt);
db_end_transaction(1);
}else{
|
| ︙ | ︙ |
Changes to src/schema.c.
| ︙ | ︙ | |||
133 134 135 136 137 138 139 140 141 142 143 144 145 146 | @ mid INTEGER REFERENCES blob, -- Manifest ID where change occurs @ pid INTEGER REFERENCES blob, -- File ID in parent manifest @ fid INTEGER REFERENCES blob, -- Changed file ID in this manifest @ fnid INTEGER REFERENCES filename -- Name of the file @ ); @ CREATE INDEX mlink_i1 ON mlink(mid); @ CREATE INDEX mlink_i2 ON mlink(fnid); @ @ -- Parent/child linkages @ -- @ CREATE TABLE plink( @ pid INTEGER REFERENCES blob, -- Parent manifest @ cid INTEGER REFERENCES blob, -- Child manifest @ isprim BOOLEAN, -- pid is the primary parent of cid | > > | 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | @ mid INTEGER REFERENCES blob, -- Manifest ID where change occurs @ pid INTEGER REFERENCES blob, -- File ID in parent manifest @ fid INTEGER REFERENCES blob, -- Changed file ID in this manifest @ fnid INTEGER REFERENCES filename -- Name of the file @ ); @ CREATE INDEX mlink_i1 ON mlink(mid); @ CREATE INDEX mlink_i2 ON mlink(fnid); @ CREATE INDEX mlink_i3 ON mlink(fid); @ CREATE INDEX mlink_i4 ON mlink(pid); @ @ -- Parent/child linkages @ -- @ CREATE TABLE plink( @ pid INTEGER REFERENCES blob, -- Parent manifest @ cid INTEGER REFERENCES blob, -- Child manifest @ isprim BOOLEAN, -- pid is the primary parent of cid |
| ︙ | ︙ | |||
158 159 160 161 162 163 164 | @ uid INTEGER REFERENCES user, @ user TEXT, @ comment TEXT @ ); @ CREATE INDEX event_i1 ON event(mtime); @ CREATE INDEX event_i2 ON event(objid); @ | | | > > | 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 | @ uid INTEGER REFERENCES user, @ user TEXT, @ comment TEXT @ ); @ CREATE INDEX event_i1 ON event(mtime); @ CREATE INDEX event_i2 ON event(objid); @ @ -- A record of phantoms @ -- @ CREATE TABLE phantom( @ rid INTEGER PRIMARY KEY -- Record ID of the phantom @ ); @ @ -- Aggregated ticket information @ -- @ CREATE TABLE tkt( @ tktid INTEGER PRIMARY KEY, -- Internal ticket ID @ fnid INTEGER REFERENCES filename, -- Name of the ticket file @ rid INTEGER REFERENCES blob, -- version of ticket file scanned |
| ︙ | ︙ |
Changes to src/xfer.c.
| ︙ | ︙ | |||
32 33 34 35 36 37 38 39 40 |
** referenced in the onremote table.
**
** Return the integer record ID of the similar record. Or return
** 0 if none is found.
*/
static int similar_record(int rid, int traceFlag){
int inCnt, outCnt;
Stmt q;
int queue[100];
| > | < | < > > | > > > > > > > > > > > > > > > > > > > > > > > > > | | | > | | | | | | | | | | | | | | | | | | | | | | | | > | 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
** referenced in the onremote table.
**
** Return the integer record ID of the similar record. Or return
** 0 if none is found.
*/
static int similar_record(int rid, int traceFlag){
int inCnt, outCnt;
int i;
Stmt q;
int queue[100];
static const char *azQuery[] = {
/* Scan the delta table first */
"SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
" FROM delta"
" WHERE rid=:x"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=srcid)"
" UNION ALL "
"SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
" FROM delta"
" WHERE srcid=:x"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=delta.rid)",
/* Then the plink table */
"SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
" FROM plink"
" WHERE cid=:x"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
" UNION ALL "
"SELECT cid, EXISTS(SELECT 1 FROM onremote WHERE rid=cid)"
" FROM plink"
" WHERE pid=:x"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=cid)",
/* Finally the mlink table */
"SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
" FROM mlink"
" WHERE fid=:x AND pid>0"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
" UNION ALL "
"SELECT fid, EXISTS(SELECT 1 FROM onremote WHERE rid=fid)"
" FROM mlink"
" WHERE pid=:x AND fid>0"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=fid)",
};
for(i=0; i<sizeof(azQuery)/sizeof(azQuery[0]); i++){
db_prepare(&q, azQuery[i]);
queue[0] = rid;
inCnt = 1;
outCnt = 0;
if( traceFlag ) printf("PASS %d\n", i+1);
while( outCnt<inCnt ){
int xid = queue[outCnt%64];
outCnt++;
db_bind_int(&q, ":x", xid);
if( traceFlag ) printf("xid=%d\n", xid);
while( db_step(&q)==SQLITE_ROW ){
int nid = db_column_int(&q, 0);
int hit = db_column_int(&q, 1);
if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
if( hit ){
db_finalize(&q);
return nid;
}
if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
int i;
for(i=0; i<inCnt && queue[i]!=nid; i++){}
if( i>=inCnt ){
queue[inCnt++] = nid;
}
}
}
db_reset(&q);
}
db_finalize(&q);
}
return 0;
}
/*
** COMMAND: test-similar-record
*/
void test_similar_record(void){
|
| ︙ | ︙ | |||
170 171 172 173 174 175 176 |
blob_zero(&uuid);
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
if( blob_size(&uuid)==0 ){
return 0;
}
content_get(rid, &content);
| > | | | < | > > | | | | > | < < < < < < < | | < < < > > | | | | | | | | | | < | 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 |
blob_zero(&uuid);
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
if( blob_size(&uuid)==0 ){
return 0;
}
content_get(rid, &content);
if( blob_size(&content)>100 ){
srcid = similar_record(rid, 0);
if( srcid ){
Blob src;
content_get(srcid, &src);
if( blob_size(&src)>100 ){
Blob delta;
blob_delta_create(&src, &content, &delta);
blob_reset(&content);
content = delta;
blob_append(&uuid, " ", 1);
blob_append(&content, "\n", 1);
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
}
blob_reset(&src);
}
}
size = blob_size(&content);
if( pOut ){
blob_appendf(pOut, "file %b %d\n", &uuid, size);
blob_append(pOut, blob_buffer(&content), size);
}else{
cgi_printf("file %b %d\n", &uuid, size);
cgi_append_content(blob_buffer(&content), size);
}
blob_reset(&content);
blob_reset(&uuid);
db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
return size;
}
/*
** Send all pending files.
|
| ︙ | ︙ | |||
465 466 467 468 469 470 471 |
nErr++;
break;
}
isPull = 1;
@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
db_multi_exec(
"INSERT OR IGNORE INTO pending(rid) "
| | | 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 |
nErr++;
break;
}
isPull = 1;
@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
db_multi_exec(
"INSERT OR IGNORE INTO pending(rid) "
"SELECT mid FROM mlink JOIN blob ON mid=rid"
);
}else
/* login USER NONCE SIGNATURE
**
** Check for a valid login. This has to happen before anything else.
*/
|
| ︙ | ︙ | |||
493 494 495 496 497 498 499 |
}
blobarray_reset(aToken, nToken);
}
/* The input message has now been processed. Generate a reply. */
if( isPush ){
Stmt q;
| > | | > > > > > > > | 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 |
}
blobarray_reset(aToken, nToken);
}
/* The input message has now been processed. Generate a reply. */
if( isPush ){
Stmt q;
int nReq = 0;
db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
while( db_step(&q)==SQLITE_ROW && nReq++ < 200 ){
const char *zUuid = db_column_text(&q, 0);
int rid = db_column_int(&q, 1);
int xid = similar_record(rid, 0);
@ gimme %s(zUuid)
if( xid ){
char *zXUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid);
@ igot %s(zXUuid);
free(zXUuid);
}
}
db_finalize(&q);
}
if( isPull ){
send_all_pending(0);
}
if( isPush || isPull ){
|
| ︙ | ︙ | |||
620 621 622 623 624 625 626 |
nMsg++;
}
if( pullFlag ){
/* Send gimme message for every phantom that we hold.
*/
Stmt q;
| | | > > > > > > | 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 |
nMsg++;
}
if( pullFlag ){
/* Send gimme message for every phantom that we hold.
*/
Stmt q;
db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
while( db_step(&q)==SQLITE_ROW && nReq<200 ){
const char *zUuid = db_column_text(&q, 0);
int rid = db_column_int(&q, 1);
int xid = similar_record(rid, 0);
blob_appendf(&send,"gimme %s\n", zUuid);
nReq++;
if( xid ){
blob_appendf(&send, "igot %z\n",
db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid));
}
}
db_finalize(&q);
}
if( pushFlag ){
/* Send the server any files that the server has requested */
nFile += send_all_pending(&send);
|
| ︙ | ︙ | |||
714 715 716 717 718 719 720 |
"SELECT cid FROM plink WHERE pid=%d", rid
);
if( db_changes()>0 ){
go = 1;
}
}
if( pullFlag && !go &&
| | | 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 |
"SELECT cid FROM plink WHERE pid=%d", rid
);
if( db_changes()>0 ){
go = 1;
}
}
if( pullFlag && !go &&
db_exists("SELECT 1 FROM phantom WHERE rid=%d", rid) ){
go = 1;
}
}else if( pullFlag ){
go = 1;
content_put(0, blob_str(&aToken[1]));
}
}else
|
| ︙ | ︙ |