Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | The xfer mechanism has been completely reworked to better support delta compression and to require fewer round-trips. The wire protocol is roughly the same but is different enough that you will need to recompile before sync will work. |
|---|---|
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
edbb332d548cc19d0f884d4c9277cbc8 |
| User & Date: | drh 2007-08-10 02:59:52.000 |
Context
|
2007-08-10
| ||
| 03:50 | More sync fixes: The previous version was not pulling new branches off of the server. This should fix that. check-in: 50150adeec user: drh tags: trunk | |
| 02:59 | The xfer mechanism has been completely reworked to better support delta compression and to require fewer round-trips. The wire protocol is roughly the same but is different enough that you will need to recompile before sync will work. check-in: edbb332d54 user: drh tags: trunk | |
| 00:08 | Complete rework of the xfer mechanism. Compiles but not yet working. check-in: 573a464cb7 user: drh tags: trunk | |
Changes
Changes to src/verify.c.
| ︙ | ︙ | |||
41 42 43 44 45 46 47 |
return; /* No way to verify phantoms */
}
blob_zero(&uuid);
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", rid);
if( blob_size(&uuid)!=UUID_SIZE ){
fossil_panic("not a valid rid: %d", rid);
}
| | | | | | | | | > | | 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
return; /* No way to verify phantoms */
}
blob_zero(&uuid);
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", rid);
if( blob_size(&uuid)!=UUID_SIZE ){
fossil_panic("not a valid rid: %d", rid);
}
if( content_get(rid, &content) ){
sha1sum_blob(&content, &hash);
blob_reset(&content);
if( blob_compare(&uuid, &hash) ){
fossil_fatal("hash of rid %d (%b) does not match its uuid (%b)",
rid, &hash, &uuid);
}
blob_reset(&hash);
}
blob_reset(&uuid);
}
/*
**
*/
static int verify_at_commit(void *notUsed){
Stmt q;
|
| ︙ | ︙ |
Changes to src/xfer.c.
| ︙ | ︙ | |||
34 35 36 37 38 39 40 |
struct Xfer {
Blob *pIn; /* Input text from the other side */
Blob *pOut; /* Compose our reply here */
Blob line; /* The current line of input */
Blob aToken[5]; /* Tokenized version of line */
Blob err; /* Error message text */
int nToken; /* Number of tokens in line */
| | > | | > > | 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
struct Xfer {
Blob *pIn; /* Input text from the other side */
Blob *pOut; /* Compose our reply here */
Blob line; /* The current line of input */
Blob aToken[5]; /* Tokenized version of line */
Blob err; /* Error message text */
int nToken; /* Number of tokens in line */
int nIGotSent; /* Number of "igot" messages sent */
int nGimmeSent; /* Number of gimme messages sent */
int nFileSent; /* Number of files sent */
int nDeltaSent; /* Number of deltas sent */
int nFileRcvd; /* Number of files received */
int nDeltaRcvd; /* Number of deltas received */
int nDanglingFile; /* Number of dangling deltas received */
int mxSend; /* Stop sending "file" with pOut reaches this size */
};
/*
** The input blob contains a UUID. Convert it into a record ID.
|
| ︙ | ︙ | |||
98 99 100 101 102 103 104 |
blob_zero(&content);
blob_zero(&hash);
blob_extract(pXfer->pIn, n, &content);
if( pXfer->nToken==4 ){
Blob src;
int srcid = rid_from_uuid(&pXfer->aToken[2], 1);
if( content_get(srcid, &src)==0 ){
| | > | | | 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 |
blob_zero(&content);
blob_zero(&hash);
blob_extract(pXfer->pIn, n, &content);
if( pXfer->nToken==4 ){
Blob src;
int srcid = rid_from_uuid(&pXfer->aToken[2], 1);
if( content_get(srcid, &src)==0 ){
content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
blob_appendf(pXfer->pOut, "gimme %b\n", &pXfer->aToken[2]);
pXfer->nGimmeSent++;
pXfer->nDanglingFile++;
return;
}
pXfer->nDeltaRcvd++;
blob_delta_apply(&src, &content, &content);
blob_reset(&src);
}else{
pXfer->nFileRcvd++;
}
sha1sum_blob(&content, &hash);
if( !blob_eq_str(&pXfer->aToken[1], blob_str(&hash), -1) ){
blob_appendf(&pXfer->err, "content does not match sha1 hash");
}
blob_reset(&hash);
rid = content_put(&content, 0, 0);
|
| ︙ | ︙ | |||
137 138 139 140 141 142 143 |
Xfer *pXfer, /* The transfer context */
int rid, /* record id of the file to send */
Blob *pContent, /* The content of the file to send */
Blob *pUuid, /* The UUID of the file to send */
int srcId /* Send as a delta against this record */
){
static const char *azQuery[] = {
| < < < < < < < < | < < < < | < < < < | | > > > > > > > > | | | | 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 |
Xfer *pXfer, /* The transfer context */
int rid, /* record id of the file to send */
Blob *pContent, /* The content of the file to send */
Blob *pUuid, /* The UUID of the file to send */
int srcId /* Send as a delta against this record */
){
static const char *azQuery[] = {
"SELECT pid FROM plink"
" WHERE cid=%d"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)",
"SELECT pid FROM mlink"
" WHERE fid=%d"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)",
};
int i;
Blob src, delta;
int size = 0;
for(i=0; srcId==0 && i<count(azQuery); i++){
srcId = db_int(0, azQuery[i], rid);
}
if( srcId>0 && content_get(srcId, &src) ){
char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", srcId);
blob_delta_create(&src, pContent, &delta);
size = blob_size(&delta);
if( size>=blob_size(pContent)-50 ){
size = 0;
}else{
blob_appendf(pXfer->pOut, "file %b %s %d\n", pUuid, zUuid, size);
blob_append(pXfer->pOut, blob_buffer(&delta), size);
/* blob_appendf(pXfer->pOut, "\n", 1); */
}
blob_reset(&delta);
free(zUuid);
blob_reset(&src);
}
return size;
}
/*
** Send the file identified by rid.
**
** The pUuid can be NULL in which case the correct UUID is computed
** from the rid.
**
** If srcId is positive, then a delta is sent against that srcId.
** If srcId is zero, then an attempt is made to find an appropriate
** file to delta against. If srcId is negative, the file is sent
** without deltaing.
*/
static void send_file(Xfer *pXfer, int rid, Blob *pUuid, int srcId){
Blob content, uuid;
int size = 0;
if( db_exists("SELECT 1 FROM sent WHERE rid=%d", rid) ){
return;
}
blob_zero(&uuid);
if( pUuid==0 ){
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
if( blob_size(&uuid)==0 ){
return;
}
pUuid = &uuid;
}
if( pXfer->mxSend<=blob_size(pXfer->pOut) ){
blob_appendf(pXfer->pOut, "igot %b\n", pUuid);
pXfer->nIGotSent++;
blob_reset(&uuid);
return;
}
content_get(rid, &content);
if( blob_size(&content)>100 ){
size = send_as_delta(pXfer, rid, &content, pUuid, srcId);
}
if( size==0 ){
int size = blob_size(&content);
blob_appendf(pXfer->pOut, "file %b %d\n", pUuid, size);
blob_append(pXfer->pOut, blob_buffer(&content), size);
pXfer->nFileSent++;
}else{
pXfer->nDeltaSent++;
}
db_multi_exec("INSERT INTO sent VALUES(%d)", rid);
blob_reset(&uuid);
}
/*
** This routine runs when either client or server is notified that
|
| ︙ | ︙ | |||
291 292 293 294 295 296 297 298 299 300 301 302 303 304 |
*/
static void request_phantoms(Xfer *pXfer){
Stmt q;
db_prepare(&q, "SELECT uuid FROM phantom JOIN blob USING(rid)");
while( db_step(&q)==SQLITE_ROW ){
const char *zUuid = db_column_text(&q, 0);
blob_appendf(pXfer->pOut, "gimme %s\n", zUuid);
}
db_finalize(&q);
}
/*
** Check the signature on an application/x-fossil payload received by
| > | 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 |
*/
static void request_phantoms(Xfer *pXfer){
Stmt q;
db_prepare(&q, "SELECT uuid FROM phantom JOIN blob USING(rid)");
while( db_step(&q)==SQLITE_ROW ){
const char *zUuid = db_column_text(&q, 0);
blob_appendf(pXfer->pOut, "gimme %s\n", zUuid);
pXfer->nGimmeSent++;
}
db_finalize(&q);
}
/*
** Check the signature on an application/x-fossil payload received by
|
| ︙ | ︙ | |||
365 366 367 368 369 370 371 |
** WEBPAGE: xfer
**
** This is the transfer handler on the server side. The transfer
** message has been uncompressed and placed in the g.cgiIn blob.
** Process this message and form an appropriate reply.
*/
void page_xfer(void){
| < > | | 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 |
** WEBPAGE: xfer
**
** This is the transfer handler on the server side. The transfer
** message has been uncompressed and placed in the g.cgiIn blob.
** Process this message and form an appropriate reply.
*/
void page_xfer(void){
int isPull = 0;
int isPush = 0;
int nErr = 0;
Xfer xfer;
memset(&xfer, 0, sizeof(xfer));
blobarray_zero(xfer.aToken, count(xfer.aToken));
cgi_set_content_type(g.zContentType);
blob_zero(&xfer.err);
xfer.pIn = &g.cgiIn;
xfer.pOut = cgi_output_blob();
xfer.mxSend = db_get_int("max-download", 1000000);
db_begin_transaction();
db_multi_exec(
"CREATE TEMP TABLE sent(rid INTEGER PRIMARY KEY);"
);
while( blob_line(xfer.pIn, &xfer.line) ){
xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
/* file UUID SIZE \n CONTENT
** file UUID DELTASRC SIZE \n CONTENT
**
** Accept a file from the client.
*/
if( blob_eq(&xfer.aToken[0], "file") ){
if( !g.okWrite ){
cgi_reset_content();
@ error not\sauthorized\sto\swrite
nErr++;
break;
}
xfer_accept_file(&xfer);
if( blob_size(&xfer.err) ){
|
| ︙ | ︙ | |||
414 415 416 417 418 419 420 |
**
** Client is requesting a file
*/
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_uuid(&xfer.aToken[1])
){
| | | | | | 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 |
**
** Client is requesting a file
*/
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_uuid(&xfer.aToken[1])
){
if( g.okRead ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
if( rid ){
send_file(&xfer, rid, &xfer.aToken[1], 0);
}
}
}else
/* igot UUID
**
** Client announces that it has a particular file
*/
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "igot")
&& blob_is_uuid(&xfer.aToken[1])
){
if( g.okWrite ){
rid_from_uuid(&xfer.aToken[1], 1);
}
}else
/* leaf UUID
**
** Client announces that it has a particular manifest
*/
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "leaf")
&& blob_is_uuid(&xfer.aToken[1])
){
if( g.okRead ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
leaf_response(&xfer, rid);
}
}else
/* pull SERVERCODE PROJECTCODE
** push SERVERCODE PROJECTCODE
**
** The client wants either send or receive
*/
if( xfer.nToken==3
&& (blob_eq(&xfer.aToken[0], "pull") || blob_eq(&xfer.aToken[0], "push"))
&& blob_is_uuid(&xfer.aToken[1])
&& blob_is_uuid(&xfer.aToken[2])
){
const char *zSCode;
const char *zPCode;
|
| ︙ | ︙ | |||
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 |
}else
/* clone
**
** The client knows nothing. Tell all.
*/
if( blob_eq(&xfer.aToken[0], "clone") ){
login_check_credentials();
if( !g.okRead || !g.okHistory ){
cgi_reset_content();
@ error not\sauthorized\sto\sclone
nErr++;
break;
}
isPull = 1;
@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
| > > > > > > > | > | | 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 |
}else
/* clone
**
** The client knows nothing. Tell all.
*/
if( blob_eq(&xfer.aToken[0], "clone") ){
int rootid;
login_check_credentials();
if( !g.okRead || !g.okHistory ){
cgi_reset_content();
@ error not\sauthorized\sto\sclone
nErr++;
break;
}
isPull = 1;
@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
rootid = db_int(0,
"SELECT pid FROM plink AS a"
" WHERE NOT EXISTS(SELECT 1 FROM plink WHERE cid=a.pid)"
);
if( rootid ){
send_file(&xfer, rootid, 0, -1);
leaf_response(&xfer, rootid);
}
}else
/* login USER NONCE SIGNATURE
**
** Check for a valid login. This has to happen before anything else.
*/
if( blob_eq(&xfer.aToken[0], "login")
&& xfer.nToken==4
){
if( disableLogin ){
g.okRead = g.okWrite = 1;
}else{
check_login(&xfer.aToken[1], &xfer.aToken[2], &xfer.aToken[3]);
}
}else
|
| ︙ | ︙ | |||
595 596 597 598 599 600 601 |
** are pulled if pullFlag is true. A full sync occurs if both are
** true.
*/
void client_sync(int pushFlag, int pullFlag, int cloneFlag){
int go = 1; /* Loop until zero */
const char *zSCode = db_get("server-code", "x");
const char *zPCode = db_get("project-code", 0);
| | | < > > < < | | | | | > > | | | < > > > > > > | > > > | 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 |
** are pulled if pullFlag is true. A full sync occurs if both are
** true.
*/
void client_sync(int pushFlag, int pullFlag, int cloneFlag){
int go = 1; /* Loop until zero */
const char *zSCode = db_get("server-code", "x");
const char *zPCode = db_get("project-code", 0);
int nMsg = 0; /* Number of messages sent or received */
int nCycle = 0; /* Number of round trips to the server */
int nFileSend = 0;
Blob send; /* Text we are sending to the server */
Blob recv; /* Reply we got back from the server */
Xfer xfer; /* Transfer data */
memset(&xfer, 0, sizeof(xfer));
xfer.pIn = &recv;
xfer.pOut = &send;
xfer.mxSend = db_get_int("max-upload", 250000);
assert( pushFlag || pullFlag || cloneFlag );
assert( !g.urlIsFile ); /* This only works for networking */
db_begin_transaction();
db_multi_exec(
"CREATE TEMP TABLE sent(rid INTEGER PRIMARY KEY);"
);
blobarray_zero(xfer.aToken, count(xfer.aToken));
blob_zero(&send);
blob_zero(&recv);
blob_zero(&xfer.err);
blob_zero(&xfer.line);
while( go ){
/* Generate a request to be sent to the server.
** Always begin with a clone, pull, or push message
*/
if( cloneFlag ){
blob_appendf(&send, "clone\n");
pushFlag = 0;
pullFlag = 0;
nMsg++;
}else if( pullFlag ){
blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
nMsg++;
request_phantoms(&xfer);
send_leaves(&xfer);
}
if( pushFlag ){
blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
nMsg++;
}
/* Exchange messages with the server */
nFileSend = xfer.nFileSent + xfer.nDeltaSent;
printf("Send: %10d bytes, %3d messages, %3d files (%d+%d)\n",
blob_size(&send), nMsg+xfer.nGimmeSent+xfer.nIGotSent,
nFileSend, xfer.nFileSent, xfer.nDeltaSent);
nMsg = 0;
xfer.nFileSent = 0;
xfer.nDeltaSent = 0;
xfer.nGimmeSent = 0;
http_exchange(&send, &recv);
blob_reset(&send);
/* Process the reply that came back from the server */
while( blob_line(&recv, &xfer.line) ){
if( blob_buffer(&xfer.line)[0]=='#' ){
continue;
}
xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
/* file UUID SIZE \n CONTENT
** file UUID DELTASRC SIZE \n CONTENT
**
** Receive a file transmitted from the other side
*/
if( blob_eq(&xfer.aToken[0],"file") ){
xfer_accept_file(&xfer);
}else
/* gimme UUID
**
** Server is requesting a file
*/
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_uuid(&xfer.aToken[1])
){
nMsg++;
if( pushFlag ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
send_file(&xfer, rid, &xfer.aToken[1], 0);
}
}else
/* igot UUID
**
** Server announces that it has a particular file
*/
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "igot")
&& blob_is_uuid(&xfer.aToken[1])
){
nMsg++;
if( pullFlag ){
if( !db_exists("SELECT 1 FROM blob WHERE uuid='%b' AND size>=0",
&xfer.aToken[1]) ){
content_put(0, blob_str(&xfer.aToken[1]), 0);
}
}
}else
/* leaf UUID
**
** Server announces that it has a particular manifest
*/
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "leaf")
&& blob_is_uuid(&xfer.aToken[1])
){
nMsg++;
if( pushFlag ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
leaf_response(&xfer, rid);
}
}else
|
| ︙ | ︙ | |||
752 753 754 755 756 757 758 759 |
blob_appendf(&xfer.err, "unknown command: %b", &xfer.aToken[0]);
}
if( blob_size(&xfer.err) ){
fossil_fatal("%b", &xfer.err);
}
blobarray_reset(xfer.aToken, xfer.nToken);
}
| > | < | > > > > > > | | < | | < > > > > > > | | > | > | | > | 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 |
blob_appendf(&xfer.err, "unknown command: %b", &xfer.aToken[0]);
}
if( blob_size(&xfer.err) ){
fossil_fatal("%b", &xfer.err);
}
blobarray_reset(xfer.aToken, xfer.nToken);
blob_reset(&xfer.line);
}
printf("Received: %10d bytes, %3d messages, %3d files (%d+%d+%d)\n",
blob_size(&recv), nMsg,
xfer.nFileRcvd + xfer.nDeltaRcvd + xfer.nDanglingFile,
xfer.nFileRcvd, xfer.nDeltaRcvd, xfer.nDanglingFile);
blob_reset(&recv);
nMsg = 0;
xfer.nFileRcvd = 0;
xfer.nDeltaRcvd = 0;
xfer.nDanglingFile = 0;
nCycle++;
go = 0;
/* If we have received one or more files on this cycle and
** we have one or more phantoms, then go for another round
*/
if(xfer.nFileRcvd+xfer.nDeltaRcvd+xfer.nDanglingFile>0
&& db_exists("SELECT 1 FROM phantom")
){
go = 1;
}
/* If we have one or more files queued to send, then go
** another round
*/
if( xfer.nFileSent+xfer.nDeltaSent>0 ){
go = 1;
}
};
http_close();
db_end_transaction(0);
}
|