Fossil

Check-in [16fbb59b96]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix the delta computation on download so that it never tries to delta a file that has the same file as both its parent and its child.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 16fbb59b96bd8731edd17d952c11fd31e78e1a43
User & Date: drh 2007-12-03 19:08:11.000
Context
2007-12-03
19:17
Send native deltas when cloning. The reduces bandwidth and reduces server load. check-in: 18fbb9c52d user: drh tags: trunk
19:08
Fix the delta computation on download so that it never tries to delta a file that has the same file as both its parent and its child. check-in: 16fbb59b96 user: drh tags: trunk
14:42
Detect delta loops and make at least one member of the loop a phantom. check-in: fb90abe5bd user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/db.c.
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    cgi_printf("<p><font color=\"red\">%h</font></p>", z);
    style_footer();
    cgi_reply();
  }else{
    fprintf(stderr, "%s: %s\n", g.argv[0], z);
  }
  db_force_rollback();
  exit(1);
  exit(1);
}

static int nBegin = 0;      /* Nesting depth of BEGIN */
static int doRollback = 0;  /* True to force a rollback */
static int nCommitHook = 0; /* Number of commit hooks */
static struct sCommitHook {







<







69
70
71
72
73
74
75

76
77
78
79
80
81
82
    cgi_printf("<p><font color=\"red\">%h</font></p>", z);
    style_footer();
    cgi_reply();
  }else{
    fprintf(stderr, "%s: %s\n", g.argv[0], z);
  }
  db_force_rollback();

  exit(1);
}

static int nBegin = 0;      /* Nesting depth of BEGIN */
static int doRollback = 0;  /* True to force a rollback */
static int nCommitHook = 0; /* Number of commit hooks */
static struct sCommitHook {
Changes to src/xfer.c.
89
90
91
92
93
94
95

96
97
98
99
100
101
102
**
** If any error occurs, write a message into pErr which has already
** be initialized to an empty string.
*/
static void xfer_accept_file(Xfer *pXfer){
  int n;
  int rid;

  Blob content, hash;
  
  if( pXfer->nToken<3 
   || pXfer->nToken>4
   || !blob_is_uuid(&pXfer->aToken[1])
   || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &n)
   || n<0







>







89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
**
** If any error occurs, write a message into pErr which has already
** be initialized to an empty string.
*/
static void xfer_accept_file(Xfer *pXfer){
  int n;
  int rid;
  int srcid = 0;
  Blob content, hash;
  
  if( pXfer->nToken<3 
   || pXfer->nToken>4
   || !blob_is_uuid(&pXfer->aToken[1])
   || !blob_is_int(&pXfer->aToken[pXfer->nToken-1], &n)
   || n<0
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
  blob_extract(pXfer->pIn, n, &content);
  if( db_exists("SELECT 1 FROM shun WHERE uuid=%B", &pXfer->aToken[1]) ){
    /* Ignore files that have been shunned */
    return;
  }
  if( pXfer->nToken==4 ){
    Blob src;
    int srcid = rid_from_uuid(&pXfer->aToken[2], 1);
    if( content_get(srcid, &src)==0 ){
      content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
      blob_appendf(pXfer->pOut, "gimme %b\n", &pXfer->aToken[2]);
      pXfer->nGimmeSent++;
      pXfer->nDanglingFile++;
      return;
    }







|







111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
  blob_extract(pXfer->pIn, n, &content);
  if( db_exists("SELECT 1 FROM shun WHERE uuid=%B", &pXfer->aToken[1]) ){
    /* Ignore files that have been shunned */
    return;
  }
  if( pXfer->nToken==4 ){
    Blob src;
    srcid = rid_from_uuid(&pXfer->aToken[2], 1);
    if( content_get(srcid, &src)==0 ){
      content_put(&content, blob_str(&pXfer->aToken[1]), srcid);
      blob_appendf(pXfer->pOut, "gimme %b\n", &pXfer->aToken[2]);
      pXfer->nGimmeSent++;
      pXfer->nDanglingFile++;
      return;
    }
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162


163
164
165
166


167
168
169
170

171
172
173
174
175
176
177
  }else{
    manifest_crosslink(rid, &content);
  }
  remote_has(rid);
}

/*
** Try to send a file as a delta.  If successful, return the number
** of bytes in the delta.  If not, return zero.
**
** If srcId is specified, use it.  If not, try to figure out a
** reasonable srcId.
*/
static int send_as_delta(
  Xfer *pXfer,            /* The transfer context */
  int rid,                /* record id of the file to send */
  Blob *pContent,         /* The content of the file to send */
  Blob *pUuid,            /* The UUID of the file to send */
  int srcId               /* Send as a delta against this record */
){
  static const char *azQuery[] = {
    "SELECT pid FROM plink"
    " WHERE cid=%d"
    "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)",



    "SELECT pid FROM mlink"
    " WHERE fid=%d"
    "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)",


  };
  int i;
  Blob src, delta;
  int size = 0;


  for(i=0; srcId==0 && i<count(azQuery); i++){
    srcId = db_int(0, azQuery[i], rid);
  }
  if( srcId>0 && content_get(srcId, &src) ){
    char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", srcId);
    blob_delta_create(&src, pContent, &delta);







|
|
<
|
|

|



|
<


|

|
>
>

|

|
>
>




>







140
141
142
143
144
145
146
147
148

149
150
151
152
153
154
155
156

157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
  }else{
    manifest_crosslink(rid, &content);
  }
  remote_has(rid);
}

/*
** Try to send a file as a delta against its parent.
** If successful, return the number of bytes in the delta.

** If we cannot generate an appropriate delta, then send
** nothing and return zero.
*/
static int send_delta_parent(
  Xfer *pXfer,            /* The transfer context */
  int rid,                /* record id of the file to send */
  Blob *pContent,         /* The content of the file to send */
  Blob *pUuid             /* The UUID of the file to send */

){
  static const char *azQuery[] = {
    "SELECT pid FROM plink x"
    " WHERE cid=%d"
    "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
    "   AND NOT EXISTS(SELECT 1 FROM plink y"
                      " WHERE y.pid=x.cid AND y.cid=x.pid)",

    "SELECT pid FROM mlink x"
    " WHERE fid=%d"
    "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
    "   AND NOT EXISTS(SELECT 1 FROM mlink y"
                     "  WHERE y.pid=x.fid AND y.fid=x.pid)"
  };
  int i;
  Blob src, delta;
  int size = 0;
  int srcId = 0;

  for(i=0; srcId==0 && i<count(azQuery); i++){
    srcId = db_int(0, azQuery[i], rid);
  }
  if( srcId>0 && content_get(srcId, &src) ){
    char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", srcId);
    blob_delta_create(&src, pContent, &delta);
185
186
187
188
189
190
191


































192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224







225
226
227
228
229
230
231
232
233
234
235
236

237
238
239
240
241
242
243
    }
    blob_reset(&delta);
    free(zUuid);
    blob_reset(&src);
  }
  return size;
}



































/*
** Send the file identified by rid.
**
** The pUuid can be NULL in which case the correct UUID is computed
** from the rid.
**
** If srcId is positive, then a delta is sent against that srcId.
** If srcId is zero, then an attempt is made to find an appropriate
** file to delta against.   If srcId is negative, the file is sent
** without deltaing.
*/
static void send_file(Xfer *pXfer, int rid, Blob *pUuid, int srcId){
  Blob content, uuid;
  int size = 0;

  if( db_exists("SELECT 1 FROM onremote WHERE rid=%d", rid) ){
     return;
  }
  blob_zero(&uuid);
  if( pUuid==0 ){
    db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
    if( blob_size(&uuid)==0 ){
      return;
    }
    pUuid = &uuid;
  }
  if( pXfer->mxSend<=blob_size(pXfer->pOut) ){
    blob_appendf(pXfer->pOut, "igot %b\n", pUuid);
    pXfer->nIGotSent++;
    blob_reset(&uuid);
    return;
  }







  content_get(rid, &content);

  if( blob_size(&content)>100 ){
    size = send_as_delta(pXfer, rid, &content, pUuid, srcId);
  }
  if( size==0 ){
    int size = blob_size(&content);
    blob_appendf(pXfer->pOut, "file %b %d\n", pUuid, size);
    blob_append(pXfer->pOut, blob_buffer(&content), size);
    pXfer->nFileSent++;
  }else{
    pXfer->nDeltaSent++;

  }
  remote_has(rid);
  blob_reset(&uuid);
}

/*
** Send a gimme message for every phantom.







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







<
<
|
|

|




















>
>
>
>
>
>
>
|

|
|
|
|
|
|
|
|
|
|
>







189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236


237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
    }
    blob_reset(&delta);
    free(zUuid);
    blob_reset(&src);
  }
  return size;
}

/*
** Try to send a file as a native delta.  
** If successful, return the number of bytes in the delta.
** If we cannot generate an appropriate delta, then send
** nothing and return zero.
*/
static int send_delta_native(
  Xfer *pXfer,            /* The transfer context */
  int rid,                /* record id of the file to send */
  Blob *pUuid             /* The UUID of the file to send */
){
  Blob src, delta;
  int size = 0;
  int srcId;

  srcId = db_int(0, "SELECT srcid FROM delta WHERE rid=%d", rid);
  if( srcId>0 ){
    blob_zero(&delta);
    db_blob(&delta, "SELECT content FROM blob WHERE rid=%d", rid);
    blob_uncompress(&delta, &delta);
    blob_zero(&src);
    db_blob(&src, "SELECT uuid FROM blob WHERE rid=%d", srcId);
    blob_appendf(pXfer->pOut, "file %b %b %d\n",
                pUuid, &src, blob_size(&delta));
    blob_append(pXfer->pOut, blob_buffer(&delta), blob_size(&delta));
    size = blob_size(&delta);
    blob_reset(&delta);
    blob_reset(&src);
  }else{
    size = 0;
  }
  return size;
}

/*
** Send the file identified by rid.
**
** The pUuid can be NULL in which case the correct UUID is computed
** from the rid.
**


** Try to send the file as a native delta if nativeDelta is true, or
** as a parent delta if nativeDelta is false.
*/
static void send_file(Xfer *pXfer, int rid, Blob *pUuid, int nativeDelta){
  Blob content, uuid;
  int size = 0;

  if( db_exists("SELECT 1 FROM onremote WHERE rid=%d", rid) ){
     return;
  }
  blob_zero(&uuid);
  if( pUuid==0 ){
    db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
    if( blob_size(&uuid)==0 ){
      return;
    }
    pUuid = &uuid;
  }
  if( pXfer->mxSend<=blob_size(pXfer->pOut) ){
    blob_appendf(pXfer->pOut, "igot %b\n", pUuid);
    pXfer->nIGotSent++;
    blob_reset(&uuid);
    return;
  }
  if( nativeDelta ){
    size = send_delta_native(pXfer, rid, pUuid);
    if( size ){
      pXfer->nDeltaSent++;
    }
  }
  if( size==0 ){
    content_get(rid, &content);

    if( !nativeDelta && blob_size(&content)>100 ){
      size = send_delta_parent(pXfer, rid, &content, pUuid);
    }
    if( size==0 ){
      int size = blob_size(&content);
      blob_appendf(pXfer->pOut, "file %b %d\n", pUuid, size);
      blob_append(pXfer->pOut, blob_buffer(&content), size);
      pXfer->nFileSent++;
    }else{
      pXfer->nDeltaSent++;
    }
  }
  remote_has(rid);
  blob_reset(&uuid);
}

/*
** Send a gimme message for every phantom.
410
411
412
413
414
415
416

417
418
419
420
421
422
423
** Process this message and form an appropriate reply.
*/
void page_xfer(void){
  int isPull = 0;
  int isPush = 0;
  int nErr = 0;
  Xfer xfer;


  memset(&xfer, 0, sizeof(xfer));
  blobarray_zero(xfer.aToken, count(xfer.aToken));
  cgi_set_content_type(g.zContentType);
  blob_zero(&xfer.err);
  xfer.pIn = &g.cgiIn;
  xfer.pOut = cgi_output_blob();







>







454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
** Process this message and form an appropriate reply.
*/
void page_xfer(void){
  int isPull = 0;
  int isPush = 0;
  int nErr = 0;
  Xfer xfer;
  int nativeDeltaFlag = 0;

  memset(&xfer, 0, sizeof(xfer));
  blobarray_zero(xfer.aToken, count(xfer.aToken));
  cgi_set_content_type(g.zContentType);
  blob_zero(&xfer.err);
  xfer.pIn = &g.cgiIn;
  xfer.pOut = cgi_output_blob();
549
550
551
552
553
554
555

556
557
558
559
560
561
562
      if( !g.okClone ){
        cgi_reset_content();
        @ error not\sauthorized\sto\sclone
        nErr++;
        break;
      }
      isPull = 1;

      @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
    }else

    /*    login  USER  NONCE  SIGNATURE
    **
    ** Check for a valid login.  This has to happen before anything else.
    ** The client can send multiple logins.  Permissions are cumulative.







>







594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
      if( !g.okClone ){
        cgi_reset_content();
        @ error not\sauthorized\sto\sclone
        nErr++;
        break;
      }
      isPull = 1;
      /* nativeDeltaFlag = 1; */
      @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
    }else

    /*    login  USER  NONCE  SIGNATURE
    **
    ** Check for a valid login.  This has to happen before anything else.
    ** The client can send multiple logins.  Permissions are cumulative.
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
    if( zCookie ){
      blob_appendf(&send, "cookie %s\n", zCookie);
    }
    
    /* Generate gimme messages for phantoms and leaf messages
    ** for all leaves.
    */
    if( pullFlag ){
      request_phantoms(&xfer);
    }
    if( pushFlag ){
      send_unsent(&xfer);
      nMsg += send_unclustered(&xfer);
    }








|







756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
    if( zCookie ){
      blob_appendf(&send, "cookie %s\n", zCookie);
    }
    
    /* Generate gimme messages for phantoms and leaf messages
    ** for all leaves.
    */
    if( pullFlag || cloneFlag ){
      request_phantoms(&xfer);
    }
    if( pushFlag ){
      send_unsent(&xfer);
      nMsg += send_unclustered(&xfer);
    }

789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
      */
      if( xfer.nToken==2
       && blob_eq(&xfer.aToken[0], "igot")
       && blob_is_uuid(&xfer.aToken[1])
      ){
        int rid = 0;
        nMsg++;
        if( pullFlag ){
          if( !db_exists("SELECT 1 FROM blob WHERE uuid='%b' AND size>=0",
                &xfer.aToken[1]) ){
            rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
            newPhantom = 1;
          }
        }
        if( rid==0 ){







|







835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
      */
      if( xfer.nToken==2
       && blob_eq(&xfer.aToken[0], "igot")
       && blob_is_uuid(&xfer.aToken[1])
      ){
        int rid = 0;
        nMsg++;
        if( pullFlag || cloneFlag ){
          if( !db_exists("SELECT 1 FROM blob WHERE uuid='%b' AND size>=0",
                &xfer.aToken[1]) ){
            rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
            newPhantom = 1;
          }
        }
        if( rid==0 ){
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
          fossil_fatal("server loop");
        }
        nMsg++;
        if( zPCode==0 ){
          zPCode = mprintf("%b", &xfer.aToken[2]);
          db_set("project-code", zPCode, 0);
        }
        cloneFlag = 0;
        pullFlag = 1;
        blob_appendf(&send, "pull %s %s\n", zSCode, zPCode);
        nMsg++;
      }else
      
      /*    cookie TEXT
      **
      ** The server might include a cookie in its reply.  The client
      ** should remember this cookie and send it back to the server







<
<
|







868
869
870
871
872
873
874


875
876
877
878
879
880
881
882
          fossil_fatal("server loop");
        }
        nMsg++;
        if( zPCode==0 ){
          zPCode = mprintf("%b", &xfer.aToken[2]);
          db_set("project-code", zPCode, 0);
        }


        blob_appendf(&send, "clone\n");
        nMsg++;
      }else
      
      /*    cookie TEXT
      **
      ** The server might include a cookie in its reply.  The client
      ** should remember this cookie and send it back to the server