Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Change the content_deltify() routine so that looks an array of candidate source artifacts and picks the one that gives the best delta. |
|---|---|
| Downloads: | Tarball | ZIP archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA3-256: |
a4047a91e3dbf58d53923e209a7a3e0c |
| User & Date: | drh 2017-09-19 01:36:46.040 |
Context
|
2017-09-19
| ||
| 01:48 | When the "showid" query parameter is given on /timeline or /finfo, show the delta-source artifact ID in addition to the artifact ID. ... (check-in: 29935c6e3e user: drh tags: trunk) | |
| 01:36 | Change the content_deltify() routine so that looks an array of candidate source artifacts and picks the one that gives the best delta. ... (check-in: a4047a91e3 user: drh tags: trunk) | |
|
2017-09-08
| ||
| 04:05 | Moved the check for iconv(3) in -liconv up within auto.def. If it happens after the checks for OpenSSL on certain macOS configurations, autosetup can be fooled into believing it doesn't need -liconv on that platform. This checkin splits up the group of cc-check-functions calls, so the moved call is now documented as to why its segregated. ... (check-in: e6f64f5eeb user: wyoung tags: trunk) | |
Changes
Changes to src/branch.c.
| ︙ | ︙ | |||
155 156 157 158 159 160 161 |
fossil_fatal("trouble committing manifest: %s", g.zErrMsg);
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", brid);
if( manifest_crosslink(brid, &branch, MC_PERMIT_HOOKS)==0 ){
fossil_fatal("%s", g.zErrMsg);
}
assert( blob_is_reset(&branch) );
| | | 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
fossil_fatal("trouble committing manifest: %s", g.zErrMsg);
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", brid);
if( manifest_crosslink(brid, &branch, MC_PERMIT_HOOKS)==0 ){
fossil_fatal("%s", g.zErrMsg);
}
assert( blob_is_reset(&branch) );
content_deltify(rootid, &brid, 1, 0);
zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", brid);
fossil_print("New branch: %s\n", zUuid);
if( g.argc==3 ){
fossil_print(
"\n"
"Note: the local check-out has not been updated to the new\n"
" branch. To begin working on the new branch, do this:\n"
|
| ︙ | ︙ |
Changes to src/checkin.c.
| ︙ | ︙ | |||
2396 2397 2398 2399 2400 2401 2402 |
fossil_print("possible unresolved merge conflict in %s\n",
blob_str(&fname));
blob_reset(&fname);
}
nrid = content_put(&content);
blob_reset(&content);
if( rid>0 ){
| | | 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 |
fossil_print("possible unresolved merge conflict in %s\n",
blob_str(&fname));
blob_reset(&fname);
}
nrid = content_put(&content);
blob_reset(&content);
if( rid>0 ){
content_deltify(rid, &nrid, 1, 0);
}
db_multi_exec("UPDATE vfile SET mrid=%d, rid=%d WHERE id=%d", nrid,nrid,id);
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
}
db_finalize(&q);
if( nConflict && !allowConflict ){
fossil_fatal("abort due to unresolved merge conflicts; "
|
| ︙ | ︙ | |||
2503 2504 2505 2506 2507 2508 2509 |
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nvid);
if( manifest_crosslink(nvid, &manifest,
dryRunFlag ? MC_NONE : MC_PERMIT_HOOKS)==0 ){
fossil_fatal("%s", g.zErrMsg);
}
assert( blob_is_reset(&manifest) );
| | | 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 |
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nvid);
if( manifest_crosslink(nvid, &manifest,
dryRunFlag ? MC_NONE : MC_PERMIT_HOOKS)==0 ){
fossil_fatal("%s", g.zErrMsg);
}
assert( blob_is_reset(&manifest) );
content_deltify(vid, &nvid, 1, 0);
zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", nvid);
db_prepare(&q, "SELECT uuid,merge FROM vmerge JOIN blob ON merge=rid"
" WHERE id=-4");
while( db_step(&q)==SQLITE_ROW ){
const char *zIntegrateUuid = db_column_text(&q, 0);
if( is_a_leaf(db_column_int(&q, 1)) ){
|
| ︙ | ︙ |
Changes to src/content.c.
| ︙ | ︙ | |||
770 771 772 773 774 775 776 |
"DELETE FROM private WHERE rid=:rid"
);
db_bind_int(&s1, ":rid", rid);
db_exec(&s1);
}
/*
| | > > | > | | > | | | | > > > | > < > > > < < | < < < | < < < < < < > | < > < > > > > > > > > > > > > > > > > > > > > > > > > > > > | | > > > > > > > > > > > > > > > > > | | < | > > | > > > > | > > > | | 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 |
"DELETE FROM private WHERE rid=:rid"
);
db_bind_int(&s1, ":rid", rid);
db_exec(&s1);
}
/*
** Try to change the storage of rid so that it is a delta from one
** of the artifacts given in aSrc[0]..aSrc[nSrc-1]. The aSrc[*] that
** gives the smallest delta is choosen.
**
** If rid is already a delta from some other place then no
** conversion occurs and this is a no-op unless force==1. If force==1,
** then nSrc must also be 1.
**
** Never generate a delta that carries a private artifact into a public
** artifact. Otherwise, when we go to send the public artifact on a
** sync operation, the other end of the sync will never be able to receive
** the source of the delta. It is OK to delta private->private and
** public->private and public->public. Just no private->public delta.
**
** If aSrc[bestSrc] is already a dleta that depends on rid, then it is
** converted to undeltaed text before the aSrc[bestSrc]->rid delta is
** created, in order to prevent a delta loop.
**
** If either rid or aSrc[i] contain less than 50 bytes, or if the
** resulting delta does not achieve a compression of at least 25%
** the rid is left untouched.
**
** Return 1 if a delta is made and 0 if no delta occurs.
*/
int content_deltify(int rid, int *aSrc, int nSrc, int force){
int s;
Blob data; /* Content of rid */
Blob src; /* Content of aSrc[i] */
Blob delta; /* Delta from aSrc[i] to rid */
Blob bestDelta; /* Best delta seen so far */
int bestSrc = 0; /* Which aSrc is the source of the best delta */
int rc = 0; /* Value to return */
int i; /* Loop variable for aSrc[] */
/* If rid is already a child (a delta) of some other artifact, return
** immediately if the force flags is false
*/
if( !force && findSrcid(rid)>0 ) return 0;
/* Get the complete content of the object to be delta-ed. If the size
** is less than 50 bytes, then there really is no point in trying to do
** a delta, so return immediately
*/
content_get(rid, &data);
if( blob_size(&data)<50 ){
/* Do not try to create a delta for objects smaller than 50 bytes */
blob_reset(&data);
return 0;
}
blob_init(&bestDelta, 0, 0);
/* Loop over all candidate delta sources */
for(i=0; i<nSrc; i++){
int srcid = aSrc[i];
if( srcid==rid ) continue;
if( content_is_private(srcid) && !content_is_private(rid) ) continue;
/* Compute all ancestors of srcid and make sure rid is not one of them.
** If rid is an ancestor of srcid, then making rid a decendent of srcid
** would create a delta loop. */
s = srcid;
while( (s = findSrcid(s))>0 ){
if( s==rid ){
content_undelta(srcid);
break;
}
}
if( s!=0 ) continue;
content_get(srcid, &src);
if( blob_size(&src)<50 ){
/* The source is smaller then 50 bytes, so don't bother trying to use it*/
blob_reset(&src);
continue;
}
blob_delta_create(&src, &data, &delta);
if( blob_size(&delta) < blob_size(&data)*0.75
&& (bestSrc<0 || blob_size(&delta)<blob_size(&bestDelta))
){
/* This is the best delta seen so far. Remember it */
blob_reset(&bestDelta);
bestDelta = delta;
bestSrc = srcid;
}else{
/* This delta is not a candidate for becoming the new parent of rid */
blob_reset(&delta);
}
blob_reset(&src);
}
/* If there is a winning candidate for the new parent of rid, then
** make that candidate the new parent now */
if( bestSrc>0 ){
Stmt s1, s2; /* Statements used to create the delta */
blob_compress(&delta, &delta);
db_prepare(&s1, "UPDATE blob SET content=:data WHERE rid=%d", rid);
db_prepare(&s2, "REPLACE INTO delta(rid,srcid)VALUES(%d,%d)", rid, bestSrc);
db_bind_blob(&s1, ":data", &bestDelta);
db_begin_transaction();
db_exec(&s1);
db_exec(&s2);
db_end_transaction(0);
db_finalize(&s1);
db_finalize(&s2);
verify_before_commit(rid);
rc = 1;
}
blob_reset(&data);
blob_reset(&bestDelta);
return rc;
}
/*
** COMMAND: test-content-deltify
**
** Usage: %fossil RID SRCID SRCID ... [-force]
**
** Convert the content at RID into a delta one of the from SRCIDs.
*/
void test_content_deltify_cmd(void){
int nSrc;
int *aSrc;
int i;
int bForce = find_option("force",0,0)!=0;
if( g.argc<3 ) usage("[--force] RID SRCID SRCID...");
aSrc = fossil_malloc( (g.argc-2)*sizeof(aSrc[0]) );
nSrc = 0;
for(i=2; i<g.argc; i++) aSrc[nSrc++] = atoi(g.argv[i]);
db_must_be_within_tree();
content_deltify(atoi(g.argv[2]), aSrc, nSrc, bForce);
}
/*
** Return true if Blob p looks like it might be a parsable control artifact.
*/
static int looks_like_control_artifact(Blob *p){
const char *z = blob_buffer(p);
|
| ︙ | ︙ |
Changes to src/event.c.
| ︙ | ︙ | |||
330 331 332 333 334 335 336 |
nrid = content_put(&event);
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
if( manifest_crosslink(nrid, &event, MC_NONE)==0 ){
db_end_transaction(1);
return 0;
}
assert( blob_is_reset(&event) );
| | | 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 |
nrid = content_put(&event);
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
if( manifest_crosslink(nrid, &event, MC_NONE)==0 ){
db_end_transaction(1);
return 0;
}
assert( blob_is_reset(&event) );
content_deltify(rid, &nrid, 1, 0);
db_end_transaction(0);
return 1;
}
/*
** WEBPAGE: technoteedit
** WEBPAGE: eventedit
|
| ︙ | ︙ |
Changes to src/json_branch.c.
| ︙ | ︙ | |||
293 294 295 296 297 298 299 |
fossil_fatal("Problem committing manifest: %s", g.zErrMsg);
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", brid);
if( manifest_crosslink(brid, &branch, MC_PERMIT_HOOKS)==0 ){
fossil_fatal("%s", g.zErrMsg);
}
assert( blob_is_reset(&branch) );
| | | 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 |
fossil_fatal("Problem committing manifest: %s", g.zErrMsg);
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", brid);
if( manifest_crosslink(brid, &branch, MC_PERMIT_HOOKS)==0 ){
fossil_fatal("%s", g.zErrMsg);
}
assert( blob_is_reset(&branch) );
content_deltify(rootid, &brid, 1, 0);
if( zNewRid ){
*zNewRid = brid;
}
/* Commit */
db_end_transaction(0);
|
| ︙ | ︙ |
Changes to src/manifest.c.
| ︙ | ︙ | |||
1244 1245 1246 1247 1248 1249 1250 |
db_bind_int(&s1, ":n", fnid);
db_bind_int(&s1, ":pfn", pfnid);
db_bind_int(&s1, ":mp", mperm);
db_bind_int(&s1, ":isaux", isPrimary==0);
db_exec(&s1);
}
if( pid && fid ){
| | | 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 |
db_bind_int(&s1, ":n", fnid);
db_bind_int(&s1, ":pfn", pfnid);
db_bind_int(&s1, ":mp", mperm);
db_bind_int(&s1, ":isaux", isPrimary==0);
db_exec(&s1);
}
if( pid && fid ){
content_deltify(pid, &fid, 1, 0);
}
}
/*
** Do a binary search to find a file in the p->aFile[] array.
**
** As an optimization, guess that the file we seek is at index p->iFile.
|
| ︙ | ︙ | |||
1433 1434 1435 1436 1437 1438 1439 |
}
/* Try to make the parent manifest a delta from the child, if that
** is an appropriate thing to do. For a new baseline, make the
** previous baseline a delta from the current baseline.
*/
if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
| | | | 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 |
}
/* Try to make the parent manifest a delta from the child, if that
** is an appropriate thing to do. For a new baseline, make the
** previous baseline a delta from the current baseline.
*/
if( (pParent->zBaseline==0)==(pChild->zBaseline==0) ){
content_deltify(pmid, &mid, 1, 0);
}else if( pChild->zBaseline==0 && pParent->zBaseline!=0 ){
content_deltify(pParent->pBaseline->rid, &mid, 1, 0);
}
/* Remember all children less than a few seconds younger than their parent,
** as we might want to fudge the times for those children.
*/
if( pChild->rDate<pParent->rDate+AGE_FUDGE_WINDOW
&& manifest_crosslink_busy
|
| ︙ | ︙ | |||
2057 2058 2059 2060 2061 2062 2063 |
prior = db_int(0,
"SELECT rid FROM tagxref"
" WHERE tagid=%d AND mtime<%.17g"
" ORDER BY mtime DESC",
tagid, p->rDate
);
if( prior ){
| | | 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 |
prior = db_int(0,
"SELECT rid FROM tagxref"
" WHERE tagid=%d AND mtime<%.17g"
" ORDER BY mtime DESC",
tagid, p->rDate
);
if( prior ){
content_deltify(prior, &rid, 1, 0);
}
if( nWiki>0 ){
zComment = mprintf("Changes to wiki page [%h]", p->zWikiTitle);
}else{
zComment = mprintf("Deleted wiki page [%h]", p->zWikiTitle);
}
search_doc_touch('w',rid,p->zWikiTitle);
|
| ︙ | ︙ | |||
2104 2105 2106 2107 2108 2109 2110 |
subsequent = db_int(0,
"SELECT rid FROM tagxref"
" WHERE tagid=%d AND mtime>=%.17g AND rid!=%d"
" ORDER BY mtime",
tagid, p->rDate, rid
);
if( prior ){
| | | | 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 |
subsequent = db_int(0,
"SELECT rid FROM tagxref"
" WHERE tagid=%d AND mtime>=%.17g AND rid!=%d"
" ORDER BY mtime",
tagid, p->rDate, rid
);
if( prior ){
content_deltify(prior, &rid, 1, 0);
if( !subsequent ){
db_multi_exec(
"DELETE FROM event"
" WHERE type='e'"
" AND tagid=%d"
" AND objid IN (SELECT rid FROM tagxref WHERE tagid=%d)",
tagid, tagid
);
}
}
if( subsequent ){
content_deltify(rid, &subsequent, 1, 0);
}else{
search_doc_touch('e',rid,0);
db_multi_exec(
"REPLACE INTO event(type,mtime,objid,tagid,user,comment,bgcolor)"
"VALUES('e',%.17g,%d,%d,%Q,%Q,"
" (SELECT value FROM tagxref WHERE tagid=%d AND rid=%d));",
p->rEventDate, rid, tagid, p->zUser, p->zComment,
|
| ︙ | ︙ |
Changes to src/rebuild.c.
| ︙ | ︙ | |||
445 446 447 448 449 450 451 452 453 454 455 456 457 |
if(!g.fQuiet && ttyOutput ){
percent_complete(1000);
fossil_print("\n");
}
return errCnt;
}
/*
** Attempt to convert more full-text blobs into delta-blobs for
** storage efficiency.
*/
void extra_deltification(void){
Stmt q;
| > > > > > > > | > > > > | | < < < | | > | > > > > > > > > < | | < < | | > | > > > > | 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 |
if(!g.fQuiet && ttyOutput ){
percent_complete(1000);
fossil_print("\n");
}
return errCnt;
}
/*
** Number of neighbors to search
*/
#define N_NEIGHBOR 5
/*
** Attempt to convert more full-text blobs into delta-blobs for
** storage efficiency.
*/
void extra_deltification(void){
Stmt q;
int aPrev[N_NEIGHBOR];
int nPrev;
int rid;
int prevfnid, fnid;
db_begin_transaction();
/* Look for manifests that have not been deltaed and try to make them
** children of one of the 5 chronologically subsequent check-ins
*/
db_prepare(&q,
"SELECT rid FROM event, blob"
" WHERE blob.rid=event.objid"
" AND event.type='ci'"
" AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
" ORDER BY event.mtime DESC"
);
nPrev = 0;
while( db_step(&q)==SQLITE_ROW ){
rid = db_column_int(&q, 0);
if( nPrev>0 ){
content_deltify(rid, aPrev, nPrev, 0);
}
if( nPrev<N_NEIGHBOR ){
aPrev[nPrev++] = rid;
}else{
int i;
for(i=0; i<N_NEIGHBOR-1; i++) aPrev[i] = aPrev[i+1];
aPrev[N_NEIGHBOR-1] = rid;
}
}
db_finalize(&q);
/* For individual files that have not been deltaed, try to find
** a parent which is an undeltaed file with the same name in a
** more recent branch.
*/
db_prepare(&q,
"SELECT blob.rid, mlink.fnid FROM blob, mlink, plink"
" WHERE NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)"
" AND mlink.fid=blob.rid"
" AND mlink.mid=plink.cid"
" AND plink.cid=mlink.mid"
" ORDER BY mlink.fnid, plink.mtime DESC"
);
prevfnid = 0;
while( db_step(&q)==SQLITE_ROW ){
rid = db_column_int(&q, 0);
fnid = db_column_int(&q, 1);
if( fnid!=prevfnid ) nPrev = 0;
if( nPrev>0 ){
content_deltify(rid, aPrev, nPrev, 0);
}
if( nPrev<N_NEIGHBOR ){
aPrev[nPrev++] = rid;
}else{
int i;
for(i=0; i<N_NEIGHBOR-1; i++) aPrev[i] = aPrev[i+1];
aPrev[N_NEIGHBOR-1] = rid;
}
}
db_finalize(&q);
db_end_transaction(0);
}
|
| ︙ | ︙ |
Changes to src/wiki.c.
| ︙ | ︙ | |||
408 409 410 411 412 413 414 |
/*
** Write a wiki artifact into the repository
*/
static void wiki_put(Blob *pWiki, int parent, int needMod){
int nrid;
if( !needMod ){
nrid = content_put_ex(pWiki, 0, 0, 0, 0);
| | | 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 |
/*
** Write a wiki artifact into the repository
*/
static void wiki_put(Blob *pWiki, int parent, int needMod){
int nrid;
if( !needMod ){
nrid = content_put_ex(pWiki, 0, 0, 0, 0);
if( parent) content_deltify(parent, &nrid, 1, 0);
}else{
nrid = content_put_ex(pWiki, 0, 0, 0, 1);
moderation_table_create();
db_multi_exec("INSERT INTO modreq(objid) VALUES(%d)", nrid);
}
db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
db_multi_exec("INSERT OR IGNORE INTO unclustered VALUES(%d);", nrid);
|
| ︙ | ︙ |