Fossil

Check-in [73bddaebb9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:The delta compress on xfer is working better now, but still needs work.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 73bddaebb9ec5d6eea26b46f56d2b82b68420f01
User & Date: drh 2007-08-09 17:42:59.000
Context
2007-08-09
19:07
Additional work on the xfer mechanism, trying to increase the use of delta compression. ... (check-in: bd3c1d0023 user: drh tags: trunk)
17:42
The delta compress on xfer is working better now, but still needs work. ... (check-in: 73bddaebb9 user: drh tags: trunk)
11:55
Fix an off-by-one error that can cause a segfault during sync. ... (check-in: 6eca3132fe user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/content.c.
194
195
196
197
198
199
200

201
202
203
204
205
206
207
208
209
210
211
212
213



214
215
216
217
218
219
220
    db_prepare(&s1,
      "UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
       g.rcvid, size, rid
    );
    blob_compress(pBlob, &cmpr);
    db_bind_blob(&s1, ":data", &cmpr);
    db_exec(&s1);

  }else{
    /* We are creating a new entry */
    db_prepare(&s1,
      "INSERT INTO blob(rcvid,size,uuid,content)"
      "VALUES(%d,%d,'%s',:data)",
       g.rcvid, size, blob_str(&hash)
    );
    if( pBlob ){
      blob_compress(pBlob, &cmpr);
      db_bind_blob(&s1, ":data", &cmpr);
    }
    db_exec(&s1);
    rid = db_last_insert_rowid();



  }


  /* Finish the transaction and cleanup */
  db_finalize(&s1);
  db_end_transaction(0);
  blob_reset(&hash);







>













>
>
>







194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
    db_prepare(&s1,
      "UPDATE blob SET rcvid=%d, size=%d, content=:data WHERE rid=%d",
       g.rcvid, size, rid
    );
    blob_compress(pBlob, &cmpr);
    db_bind_blob(&s1, ":data", &cmpr);
    db_exec(&s1);
    db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
  }else{
    /* We are creating a new entry */
    db_prepare(&s1,
      "INSERT INTO blob(rcvid,size,uuid,content)"
      "VALUES(%d,%d,'%s',:data)",
       g.rcvid, size, blob_str(&hash)
    );
    if( pBlob ){
      blob_compress(pBlob, &cmpr);
      db_bind_blob(&s1, ":data", &cmpr);
    }
    db_exec(&s1);
    rid = db_last_insert_rowid();
    if( !pBlob ){
      db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
    }
  }


  /* Finish the transaction and cleanup */
  db_finalize(&s1);
  db_end_transaction(0);
  blob_reset(&hash);
Changes to src/delta.c.
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
    landmark[hv] = i/NHASH;
  }

  /* Begin scanning the target file and generating copy commands and
  ** literal sections of the delta.
  */
  base = 0;    /* We have already generated everything before zOut[base] */
  while( base<lenOut-NHASH ){
    int iSrc, iBlock;
    unsigned int bestCnt, bestOfst, bestLitsz;
    hash_init(&h, &zOut[base]);
    i = 0;     /* Trying to match a landmark against zOut[base+i] */
    bestCnt = 0;
    while( 1 ){
      int hv;







|







331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
    landmark[hv] = i/NHASH;
  }

  /* Begin scanning the target file and generating copy commands and
  ** literal sections of the delta.
  */
  base = 0;    /* We have already generated everything before zOut[base] */
  while( base+NHASH<lenOut ){
    int iSrc, iBlock;
    unsigned int bestCnt, bestOfst, bestLitsz;
    hash_init(&h, &zOut[base]);
    i = 0;     /* Trying to match a landmark against zOut[base+i] */
    bestCnt = 0;
    while( 1 ){
      int hv;
Changes to src/rebuild.c.
58
59
60
61
62
63
64
65
66
67


68
69
70
71



72
73
74
75
76
77
78
       " AND name NOT IN ('blob','delta','rcvfrom','user','config')");
    if( zTable==0 ) break;
    db_multi_exec("DROP TABLE %Q", zTable);
    free(zTable);
  }
  db_multi_exec(zRepositorySchema2);

  db_prepare(&s, "SELECT rid FROM blob");
  while( db_step(&s)==SQLITE_ROW ){
    int rid = db_column_int(&s, 0);


    Blob content;
    content_get(rid, &content);
    manifest_crosslink(rid, &content);
    blob_reset(&content);



  }

  if( errCnt && !forceFlag ){
    printf("%d errors. Rolling back changes. Use --force to force a commit.\n",
            errCnt);
    db_end_transaction(1);
  }else{







|


>
>
|
|
|
|
>
>
>







58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
       " AND name NOT IN ('blob','delta','rcvfrom','user','config')");
    if( zTable==0 ) break;
    db_multi_exec("DROP TABLE %Q", zTable);
    free(zTable);
  }
  db_multi_exec(zRepositorySchema2);

  db_prepare(&s, "SELECT rid, size FROM blob");
  while( db_step(&s)==SQLITE_ROW ){
    int rid = db_column_int(&s, 0);
    int size = db_column_int(&s, 1);
    if( size>=0 ){
      Blob content;
      content_get(rid, &content);
      manifest_crosslink(rid, &content);
      blob_reset(&content);
    }else{
      db_multi_exec("INSERT INTO phantom VALUES(%d)", rid);
    }
  }

  if( errCnt && !forceFlag ){
    printf("%d errors. Rolling back changes. Use --force to force a commit.\n",
            errCnt);
    db_end_transaction(1);
  }else{
Changes to src/schema.c.
133
134
135
136
137
138
139


140
141
142
143
144
145
146
@   mid INTEGER REFERENCES blob,        -- Manifest ID where change occurs
@   pid INTEGER REFERENCES blob,        -- File ID in parent manifest
@   fid INTEGER REFERENCES blob,        -- Changed file ID in this manifest
@   fnid INTEGER REFERENCES filename    -- Name of the file
@ );
@ CREATE INDEX mlink_i1 ON mlink(mid);
@ CREATE INDEX mlink_i2 ON mlink(fnid);


@
@ -- Parent/child linkages
@ --
@ CREATE TABLE plink(
@   pid INTEGER REFERENCES blob,    -- Parent manifest
@   cid INTEGER REFERENCES blob,    -- Child manifest
@   isprim BOOLEAN,                 -- pid is the primary parent of cid







>
>







133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
@   mid INTEGER REFERENCES blob,        -- Manifest ID where change occurs
@   pid INTEGER REFERENCES blob,        -- File ID in parent manifest
@   fid INTEGER REFERENCES blob,        -- Changed file ID in this manifest
@   fnid INTEGER REFERENCES filename    -- Name of the file
@ );
@ CREATE INDEX mlink_i1 ON mlink(mid);
@ CREATE INDEX mlink_i2 ON mlink(fnid);
@ CREATE INDEX mlink_i3 ON mlink(fid);
@ CREATE INDEX mlink_i4 ON mlink(pid);
@
@ -- Parent/child linkages
@ --
@ CREATE TABLE plink(
@   pid INTEGER REFERENCES blob,    -- Parent manifest
@   cid INTEGER REFERENCES blob,    -- Child manifest
@   isprim BOOLEAN,                 -- pid is the primary parent of cid
158
159
160
161
162
163
164
165
166
167


168
169
170
171
172
173
174
@   uid INTEGER REFERENCES user,
@   user TEXT,
@   comment TEXT
@ );
@ CREATE INDEX event_i1 ON event(mtime);
@ CREATE INDEX event_i2 ON event(objid);
@
@ -- Make sure reading DELTA by SRCID is efficient
@ --
@ CREATE INDEX IF NOT EXISTS delta_srcid ON delta(srcid, rid);


@
@ -- Aggregated ticket information
@ --
@ CREATE TABLE tkt(
@   tktid INTEGER PRIMARY KEY,           -- Internal ticket ID
@   fnid INTEGER REFERENCES filename,    -- Name of the ticket file
@   rid INTEGER REFERENCES blob,         -- version of ticket file scanned







|

|
>
>







160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
@   uid INTEGER REFERENCES user,
@   user TEXT,
@   comment TEXT
@ );
@ CREATE INDEX event_i1 ON event(mtime);
@ CREATE INDEX event_i2 ON event(objid);
@
@ -- A record of phantoms
@ --
@ CREATE TABLE phantom(
@   rid INTEGER PRIMARY KEY         -- Record ID of the phantom
@ );
@
@ -- Aggregated ticket information
@ --
@ CREATE TABLE tkt(
@   tktid INTEGER PRIMARY KEY,           -- Internal ticket ID
@   fnid INTEGER REFERENCES filename,    -- Name of the ticket file
@   rid INTEGER REFERENCES blob,         -- version of ticket file scanned
Changes to src/xfer.c.
32
33
34
35
36
37
38

39
40
41
42
43
44
45
46
47

48
49
50
51

52

























53
54
55

56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

80
81
82
83
84
85
86
** referenced in the onremote table.
**
** Return the integer record ID of the similar record.  Or return
** 0 if none is found.
*/
static int similar_record(int rid, int traceFlag){
  int inCnt, outCnt;

  Stmt q;
  int queue[100];

return 0;

  db_prepare(&q,
      "SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
      "  FROM delta"
      " WHERE rid=:x"

      " UNION ALL "
      "SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
      "  FROM delta"
      " WHERE srcid=:x"

  );

























  queue[0] = rid;
  inCnt = 1;
  outCnt = 0;

  while( outCnt<inCnt ){
    int xid = queue[outCnt%64];
    outCnt++;
    db_bind_int(&q, ":x", xid);
    if( traceFlag ) printf("xid=%d\n", xid);
    while( db_step(&q)==SQLITE_ROW ){
      int nid = db_column_int(&q, 0);
      int hit = db_column_int(&q, 1);
      if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
      if( hit  ){
        db_finalize(&q);
        return nid;
      }
      if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
        int i;
        for(i=0; i<inCnt && queue[i]!=nid; i++){}
        if( i>=inCnt ){
          queue[inCnt++] = nid;
        }
      }
    }
    db_reset(&q);
  }
  db_finalize(&q);

  return 0;
}

/*
** COMMAND: test-similar-record
*/
void test_similar_record(void){







>


|
<
|
<



>




>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>







32
33
34
35
36
37
38
39
40
41
42

43

44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
** referenced in the onremote table.
**
** Return the integer record ID of the similar record.  Or return
** 0 if none is found.
*/
static int similar_record(int rid, int traceFlag){
  int inCnt, outCnt;
  int i;
  Stmt q;
  int queue[100];
  static const char *azQuery[] = {

      /* Scan the delta table first */

      "SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
      "  FROM delta"
      " WHERE rid=:x"
      "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=srcid)"
      " UNION ALL "
      "SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
      "  FROM delta"
      " WHERE srcid=:x"
      "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=delta.rid)",

      /* Then the plink table */
      "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
      "  FROM plink"
      " WHERE cid=:x"
      "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
      " UNION ALL "
      "SELECT cid, EXISTS(SELECT 1 FROM onremote WHERE rid=cid)"
      "  FROM plink"
      " WHERE pid=:x"
      "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=cid)",

      /* Finally the mlink table */
      "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
      "  FROM mlink"
      " WHERE fid=:x AND pid>0"
      "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
      " UNION ALL "
      "SELECT fid, EXISTS(SELECT 1 FROM onremote WHERE rid=fid)"
      "  FROM mlink"
      " WHERE pid=:x AND fid>0"
      "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=fid)",
  };

  for(i=0; i<sizeof(azQuery)/sizeof(azQuery[0]); i++){
    db_prepare(&q, azQuery[i]);
    queue[0] = rid;
    inCnt = 1;
    outCnt = 0;
    if( traceFlag ) printf("PASS %d\n", i+1);
    while( outCnt<inCnt ){
      int xid = queue[outCnt%64];
      outCnt++;
      db_bind_int(&q, ":x", xid);
      if( traceFlag ) printf("xid=%d\n", xid);
      while( db_step(&q)==SQLITE_ROW ){
        int nid = db_column_int(&q, 0);
        int hit = db_column_int(&q, 1);
        if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
        if( hit  ){
          db_finalize(&q);
          return nid;
        }
        if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
          int i;
          for(i=0; i<inCnt && queue[i]!=nid; i++){}
          if( i>=inCnt ){
            queue[inCnt++] = nid;
          }
        }
      }
      db_reset(&q);
    }
    db_finalize(&q);
  }
  return 0;
}

/*
** COMMAND: test-similar-record
*/
void test_similar_record(void){
170
171
172
173
174
175
176

177
178
179
180
181


182
183
184
185

186
187
188
189
190
191
192
193
194
195
196
197
198


199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
  blob_zero(&uuid);
  db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
  if( blob_size(&uuid)==0 ){
    return 0;
  }
  content_get(rid, &content);


  srcid = similar_record(rid, 0);
  if( srcid ){
    Blob src, delta;
    Blob srcuuid;
    content_get(srcid, &src);


    blob_delta_create(&src, &content, &delta);
    blob_reset(&src);
    blob_reset(&content);
    blob_zero(&srcuuid);

    db_blob(&srcuuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
    size = blob_size(&delta);
    if( pOut ){
      blob_appendf(pOut, "file %b %b %d\n", &uuid, &srcuuid, size);
      blob_append(pOut, blob_buffer(&delta), size);
    }else{
      cgi_printf("file %b %b %d\n", &uuid, &srcuuid, size);
      cgi_append_content(blob_buffer(&delta), size);
    }
    blob_reset(&delta);
    blob_reset(&srcuuid);
    blob_reset(&uuid);
  }else{


    size = blob_size(&content);
    if( pOut ){
      blob_appendf(pOut, "file %b %d\n", &uuid, size);
      blob_append(pOut, blob_buffer(&content), size);
    }else{
      cgi_printf("file %b %d\n", &uuid, size);
      cgi_append_content(blob_buffer(&content), size);
    }
    blob_reset(&content);
    blob_reset(&uuid);
  }
  db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
  return size;
}


/*
** Send all pending files.







>
|
|
|
<
|
>
>
|
|
|
|
>
|
<
<
<
<
<
<
<
|
|
<
<
<
>
>
|
|
|
|
|
|
|
|
|
|
<







198
199
200
201
202
203
204
205
206
207
208

209
210
211
212
213
214
215
216
217







218
219



220
221
222
223
224
225
226
227
228
229
230
231

232
233
234
235
236
237
238
  blob_zero(&uuid);
  db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
  if( blob_size(&uuid)==0 ){
    return 0;
  }
  content_get(rid, &content);

  if( blob_size(&content)>100 ){
    srcid = similar_record(rid, 0);
    if( srcid ){
      Blob src;

      content_get(srcid, &src);
      if( blob_size(&src)>100 ){
        Blob delta;
        blob_delta_create(&src, &content, &delta);
        blob_reset(&content);
        content = delta;
        blob_append(&uuid, " ", 1);
        blob_append(&content, "\n", 1);
        db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);







      }
      blob_reset(&src);



    }
  }
  size = blob_size(&content);
  if( pOut ){
    blob_appendf(pOut, "file %b %d\n", &uuid, size);
    blob_append(pOut, blob_buffer(&content), size);
  }else{
    cgi_printf("file %b %d\n", &uuid, size);
    cgi_append_content(blob_buffer(&content), size);
  }
  blob_reset(&content);
  blob_reset(&uuid);

  db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
  return size;
}


/*
** Send all pending files.
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
        nErr++;
        break;
      }
      isPull = 1;
      @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
      db_multi_exec(
        "INSERT OR IGNORE INTO pending(rid) "
        "SELECT rid FROM blob WHERE size>=0"
      );
    }else

    /*    login  USER  NONCE  SIGNATURE
    **
    ** Check for a valid login.  This has to happen before anything else.
    */







|







487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
        nErr++;
        break;
      }
      isPull = 1;
      @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
      db_multi_exec(
        "INSERT OR IGNORE INTO pending(rid) "
        "SELECT mid FROM mlink JOIN blob ON mid=rid"
      );
    }else

    /*    login  USER  NONCE  SIGNATURE
    **
    ** Check for a valid login.  This has to happen before anything else.
    */
493
494
495
496
497
498
499

500
501
502


503





504
505
506
507
508
509
510
    }
    blobarray_reset(aToken, nToken);
  }

  /* The input message has now been processed.  Generate a reply. */
  if( isPush ){
    Stmt q;

    db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
    while( db_step(&q)==SQLITE_ROW ){
      const char *zUuid = db_column_text(&q, 0);


      @ gimme %s(zUuid)





    }
    db_finalize(&q);
  }
  if( isPull ){
    send_all_pending(0);
  }
  if( isPush || isPull ){







>
|
|

>
>

>
>
>
>
>







515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
    }
    blobarray_reset(aToken, nToken);
  }

  /* The input message has now been processed.  Generate a reply. */
  if( isPush ){
    Stmt q;
    int nReq = 0;
    db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
    while( db_step(&q)==SQLITE_ROW && nReq++ < 200 ){
      const char *zUuid = db_column_text(&q, 0);
      int rid = db_column_int(&q, 1);
      int xid = similar_record(rid, 0);
      @ gimme %s(zUuid)
      if( xid ){
        char *zXUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid);
        @ igot %s(zXUuid);
        free(zXUuid);
      }
    }
    db_finalize(&q);
  }
  if( isPull ){
    send_all_pending(0);
  }
  if( isPush || isPull ){
620
621
622
623
624
625
626
627
628
629


630
631




632
633
634
635
636
637
638
      nMsg++;
    }

    if( pullFlag ){
      /* Send gimme message for every phantom that we hold.
      */
      Stmt q;
      db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
      while( db_step(&q)==SQLITE_ROW ){
        const char *zUuid = db_column_text(&q, 0);


        blob_appendf(&send,"gimme %s\n", zUuid);
        nReq++;




      }
      db_finalize(&q);
    }

    if( pushFlag ){
      /* Send the server any files that the server has requested */
      nFile += send_all_pending(&send);







|
|

>
>


>
>
>
>







650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
      nMsg++;
    }

    if( pullFlag ){
      /* Send gimme message for every phantom that we hold.
      */
      Stmt q;
      db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
      while( db_step(&q)==SQLITE_ROW && nReq<200 ){
        const char *zUuid = db_column_text(&q, 0);
        int rid = db_column_int(&q, 1);
        int xid = similar_record(rid, 0);
        blob_appendf(&send,"gimme %s\n", zUuid);
        nReq++;
        if( xid ){
          blob_appendf(&send, "igot %z\n",
             db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid));
        }
      }
      db_finalize(&q);
    }

    if( pushFlag ){
      /* Send the server any files that the server has requested */
      nFile += send_all_pending(&send);
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
              "SELECT cid FROM plink WHERE pid=%d", rid
            );
            if( db_changes()>0 ){
              go = 1;
            }
          }
          if( pullFlag && !go && 
              db_exists("SELECT 1 FROM blob WHERE rid=%d AND size<0", rid) ){
            go = 1;
          }
        }else if( pullFlag ){
          go = 1;
          content_put(0, blob_str(&aToken[1]));
        }
      }else







|







750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
              "SELECT cid FROM plink WHERE pid=%d", rid
            );
            if( db_changes()>0 ){
              go = 1;
            }
          }
          if( pullFlag && !go && 
              db_exists("SELECT 1 FROM phantom WHERE rid=%d", rid) ){
            go = 1;
          }
        }else if( pullFlag ){
          go = 1;
          content_put(0, blob_str(&aToken[1]));
        }
      }else