Fossil

Check-in [18c0ab6bf1]
Login

Check-in [18c0ab6bf1]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Get "fossil rebuild" to work on an indexed database, and automatically recompute the index.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 18c0ab6bf1f79a5c71e897cd7e298b638253855e
User & Date: drh 2015-02-03 16:54:17.528
Context
2015-02-03
17:48
merge [fsl_setting_proxy] to fix [5e35dd60cf7410af64d4] ... (check-in: 00b62e0595 user: bch tags: trunk)
16:54
Get "fossil rebuild" to work on an indexed database, and automatically recompute the index. ... (check-in: 18c0ab6bf1 user: drh tags: trunk)
16:40
Take out the "test-fts" command and replace it with "fts-config" which has a more user-friendly format. Add "fts-config" to the "all" command too. ... (check-in: 7b6adfbf92 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/allrepo.c.
227
228
229
230
231
232
233


234
235
236
237
238
239
240
    collect_argument(&extra, "noverify",0);
    collect_argument_value(&extra, "pagesize");
    collect_argument(&extra, "vacuum",0);
    collect_argument(&extra, "deanalyze",0);
    collect_argument(&extra, "analyze",0);
    collect_argument(&extra, "wal",0);
    collect_argument(&extra, "stats",0);


  }else if( strncmp(zCmd, "setting", n)==0 ){
    zCmd = "setting -R";
    collect_argv(&extra, 3);
  }else if( strncmp(zCmd, "unset", n)==0 ){
    zCmd = "unset -R";
    collect_argv(&extra, 3);
  }else if( strncmp(zCmd, "fts-config", n)==0 ){







>
>







227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
    collect_argument(&extra, "noverify",0);
    collect_argument_value(&extra, "pagesize");
    collect_argument(&extra, "vacuum",0);
    collect_argument(&extra, "deanalyze",0);
    collect_argument(&extra, "analyze",0);
    collect_argument(&extra, "wal",0);
    collect_argument(&extra, "stats",0);
    collect_argument(&extra, "index",0);
    collect_argument(&extra, "no-index",0);
  }else if( strncmp(zCmd, "setting", n)==0 ){
    zCmd = "setting -R";
    collect_argv(&extra, 3);
  }else if( strncmp(zCmd, "unset", n)==0 ){
    zCmd = "unset -R";
    collect_argv(&extra, 3);
  }else if( strncmp(zCmd, "fts-config", n)==0 ){
Changes to src/rebuild.c.
531
532
533
534
535
536
537


538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554

555
556
557
558
559
560
561
**   --pagesize N  Set the database pagesize to N. (512..65536 and power of 2)
**   --randomize   Scan artifacts in a random order
**   --vacuum      Run VACUUM on the database after rebuilding
**   --deanalyze   Remove ANALYZE tables from the database
**   --analyze     Run ANALYZE on the database after rebuilding
**   --wal         Set Write-Ahead-Log journalling mode on the database
**   --stats       Show artifact statistics after rebuilding


**
** See also: deconstruct, reconstruct
*/
void rebuild_database(void){
  int forceFlag;
  int randomizeFlag;
  int errCnt;
  int omitVerify;
  int doClustering;
  const char *zPagesize;
  int newPagesize = 0;
  int activateWal;
  int runVacuum;
  int runDeanalyze;
  int runAnalyze;
  int runCompress;
  int showStats;


  omitVerify = find_option("noverify",0,0)!=0;
  forceFlag = find_option("force","f",0)!=0;
  randomizeFlag = find_option("randomize", 0, 0)!=0;
  doClustering = find_option("cluster", 0, 0)!=0;
  runVacuum = find_option("vacuum",0,0)!=0;
  runDeanalyze = find_option("deanalyze",0,0)!=0;







>
>

















>







531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
**   --pagesize N  Set the database pagesize to N. (512..65536 and power of 2)
**   --randomize   Scan artifacts in a random order
**   --vacuum      Run VACUUM on the database after rebuilding
**   --deanalyze   Remove ANALYZE tables from the database
**   --analyze     Run ANALYZE on the database after rebuilding
**   --wal         Set Write-Ahead-Log journalling mode on the database
**   --stats       Show artifact statistics after rebuilding
**   --index       Always add in the full-text search index
**   --no-index    Always omit the full-text search index
**
** See also: deconstruct, reconstruct
*/
void rebuild_database(void){
  int forceFlag;
  int randomizeFlag;
  int errCnt;
  int omitVerify;
  int doClustering;
  const char *zPagesize;
  int newPagesize = 0;
  int activateWal;
  int runVacuum;
  int runDeanalyze;
  int runAnalyze;
  int runCompress;
  int showStats;
  int runReindex;

  omitVerify = find_option("noverify",0,0)!=0;
  forceFlag = find_option("force","f",0)!=0;
  randomizeFlag = find_option("randomize", 0, 0)!=0;
  doClustering = find_option("cluster", 0, 0)!=0;
  runVacuum = find_option("vacuum",0,0)!=0;
  runDeanalyze = find_option("deanalyze",0,0)!=0;
578
579
580
581
582
583
584



585
586
587
588
589

590
591
592
593
594
595
596
    db_find_and_open_repository(OPEN_ANY_SCHEMA, 0);
    if( g.argc!=2 ){
      usage("?REPOSITORY-FILENAME?");
    }
    db_close(1);
    db_open_repository(g.zRepositoryName);
  }



  
  /* We should be done with options.. */
  verify_all_options();

  db_begin_transaction();

  ttyOutput = 1;
  errCnt = rebuild_db(randomizeFlag, 1, doClustering);
  reconstruct_private_table();
  db_multi_exec(
    "REPLACE INTO config(name,value,mtime) VALUES('content-schema',%Q,now());"
    "REPLACE INTO config(name,value,mtime) VALUES('aux-schema',%Q,now());"
    "REPLACE INTO config(name,value,mtime) VALUES('rebuilt',%Q,now());",







>
>
>





>







581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
    db_find_and_open_repository(OPEN_ANY_SCHEMA, 0);
    if( g.argc!=2 ){
      usage("?REPOSITORY-FILENAME?");
    }
    db_close(1);
    db_open_repository(g.zRepositoryName);
  }
  runReindex = search_index_exists();
  if( find_option("index",0,0)!=0 ) runReindex = 1;
  if( find_option("no-index",0,0)!=0 ) runReindex = 0;
  
  /* We should be done with options.. */
  verify_all_options();

  db_begin_transaction();
  search_drop_index();
  ttyOutput = 1;
  errCnt = rebuild_db(randomizeFlag, 1, doClustering);
  reconstruct_private_table();
  db_multi_exec(
    "REPLACE INTO config(name,value,mtime) VALUES('content-schema',%Q,now());"
    "REPLACE INTO config(name,value,mtime) VALUES('aux-schema',%Q,now());"
    "REPLACE INTO config(name,value,mtime) VALUES('rebuilt',%Q,now());",
632
633
634
635
636
637
638

639
640
641
642
643
644
645
      db_multi_exec("VACUUM");
      fossil_print("done\n");
    }
    if( activateWal ){
      db_multi_exec("PRAGMA journal_mode=WAL;");
    }
  }

  if( showStats ){
    static const struct { int idx; const char *zLabel; } aStat[] = {
       { CFTYPE_ANY,       "Artifacts:" },
       { CFTYPE_MANIFEST,  "Manifests:" },
       { CFTYPE_CLUSTER,   "Clusters:" },
       { CFTYPE_CONTROL,   "Tags:" },
       { CFTYPE_WIKI,      "Wikis:" },







>







639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
      db_multi_exec("VACUUM");
      fossil_print("done\n");
    }
    if( activateWal ){
      db_multi_exec("PRAGMA journal_mode=WAL;");
    }
  }
  if( runReindex ) search_rebuild_index();
  if( showStats ){
    static const struct { int idx; const char *zLabel; } aStat[] = {
       { CFTYPE_ANY,       "Artifacts:" },
       { CFTYPE_MANIFEST,  "Manifests:" },
       { CFTYPE_CLUSTER,   "Clusters:" },
       { CFTYPE_CONTROL,   "Tags:" },
       { CFTYPE_WIKI,      "Wikis:" },
Changes to src/search.c.
1134
1135
1136
1137
1138
1139
1140

1141
1142
1143
1144
1145

1146
1147
1148
1149

1150
1151
1152
1153
1154
1155
1156
1157

1158
1159
1160
1161
1162
1163
1164
1165
@ DROP VIEW IF EXISTS "%w".ftscontent;
@ DROP TABLE IF EXISTS "%w".ftsdocs;
;

/*
** Create or drop the tables associated with a full-text index.
*/

void search_create_index(void){
  const char *zDb = db_name("repository");
  search_sql_setup(g.db);
  db_multi_exec(zFtsSchema/*works-like:"%w%w%w%w%w"*/,
       zDb, zDb, zDb, zDb, zDb);

}
void search_drop_index(void){
  const char *zDb = db_name("repository");
  db_multi_exec(zFtsDrop/*works-like:"%w%w%w"*/, zDb, zDb, zDb);

}

/*
** Return true if the full-text search index exists
*/
int search_index_exists(void){
  static int fExists = -1;
  if( fExists<0 ) fExists = db_table_exists("repository","ftsdocs");

  return fExists;
}

/*
** Fill the FTSDOCS table with unindexed entries for everything
** in the repository.  This uses INSERT OR IGNORE so entries already
** in FTSDOCS are unchanged.
*/







>





>




>






|
|
>
|







1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
@ DROP VIEW IF EXISTS "%w".ftscontent;
@ DROP TABLE IF EXISTS "%w".ftsdocs;
;

/*
** Create or drop the tables associated with a full-text index.
*/
static int searchIdxExists = -1;
void search_create_index(void){
  const char *zDb = db_name("repository");
  search_sql_setup(g.db);
  db_multi_exec(zFtsSchema/*works-like:"%w%w%w%w%w"*/,
       zDb, zDb, zDb, zDb, zDb);
  searchIdxExists = 1;
}
void search_drop_index(void){
  const char *zDb = db_name("repository");
  db_multi_exec(zFtsDrop/*works-like:"%w%w%w"*/, zDb, zDb, zDb);
  searchIdxExists = 0;
}

/*
** Return true if the full-text search index exists
*/
int search_index_exists(void){
  if( searchIdxExists<0 ){
    searchIdxExists = db_table_exists("repository","ftsdocs");
  }
  return searchIdxExists;
}

/*
** Fill the FTSDOCS table with unindexed entries for everything
** in the repository.  This uses INSERT OR IGNORE so entries already
** in FTSDOCS are unchanged.
*/
1359
1360
1361
1362
1363
1364
1365












1366
1367
1368
1369
1370
1371
1372
  if( srchFlags & SRCH_TKT ){
    search_update_ticket_index();
  }
  if( srchFlags & SRCH_WIKI ){
    search_update_wiki_index();
  }
}













/*
** COMMAND: fts-config*
**
** Usage: fossil fts-config ?SUBCOMMAND? ?ARGUMENT?
**
** The "fossil fts-config" command configures the full-text search capabilities







>
>
>
>
>
>
>
>
>
>
>
>







1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
  if( srchFlags & SRCH_TKT ){
    search_update_ticket_index();
  }
  if( srchFlags & SRCH_WIKI ){
    search_update_wiki_index();
  }
}

/*
** Construct, prepopulate, and then update the full-text index.
*/
void search_rebuild_index(void){
  fossil_print("rebuilding the search index...");
  fflush(stdout);
  search_create_index();
  search_fill_index();
  search_update_index(search_restrict(SRCH_ALL));
  fossil_print(" done\n");
}

/*
** COMMAND: fts-config*
**
** Usage: fossil fts-config ?SUBCOMMAND? ?ARGUMENT?
**
** The "fossil fts-config" command configures the full-text search capabilities
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
  }

  /* destroy or rebuild the index, if requested */
  if( iAction>=1 ){
    search_drop_index();
  }
  if( iAction>=2 ){
    fossil_print("rebuilding the search index...");
    fflush(stdout);
    search_create_index();
    search_fill_index();
    search_update_index(search_restrict(SRCH_ALL));
    fossil_print(" done\n");
  }

  /* Always show the status before ending */
  for(i=0; i<ArraySize(aSetng); i++){
    fossil_print("%-16s %s\n", aSetng[i].zName, 
       db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
  }







<
<
<
|
<
<







1457
1458
1459
1460
1461
1462
1463



1464


1465
1466
1467
1468
1469
1470
1471
  }

  /* destroy or rebuild the index, if requested */
  if( iAction>=1 ){
    search_drop_index();
  }
  if( iAction>=2 ){



    search_rebuild_index();


  }

  /* Always show the status before ending */
  for(i=0; i<ArraySize(aSetng); i++){
    fossil_print("%-16s %s\n", aSetng[i].zName, 
       db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
  }