Check-in [53d1f053ad]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Many improvements to the indexed search debugging logic.
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 53d1f053ad9271639bfc998ead403fcc55aa877f54171825a4725d0baca1c421
User & Date: drh 2020-04-04 19:34:29.234
Context
2020-04-05
19:02
Improved processing of timestamp strings used as check-in identifiers. The ISO8601 date without punctuation (digits only) is now accept on the tag:date format. Also, incomplete ISO8601 date/time values are rounding up instead of rounded down, such that the match on 20200101 actually finds a check-in on 2020-01-01 (if any) rather than the last chec... check-in: a5f3103111 user: drh tags: trunk
2020-04-04
19:34
Many improvements to the indexed search debugging logic. check-in: 53d1f053ad user: drh tags: trunk
18:03
Minor comment fixes in the search logic. No changes to code. check-in: 4d35160866 user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/search.c.
1850
1851
1852
1853
1854
1855
1856
1857



1858
1859
1860
1861
1862
1863
1864




1865
1866
1867
1868
1869
1870
1871
**     stemmer (on|off)   Turn the Porter stemmer on or off for indexed
**                        search.  (Unindexed search is never stemmed.)
**
** The current search settings are displayed after any changes are applied.
** Run this command with no arguments to simply see the settings.
*/
void fts_config_cmd(void){
  static const struct { int iCmd; const char *z; } aCmd[] = {



     { 1,  "reindex"  },
     { 2,  "index"    },
     { 3,  "disable"  },
     { 4,  "enable"   },
     { 5,  "stemmer"  },
  };
  static const struct { const char *zSetting; const char *zName; const char *zSw; } aSetng[] = {




     { "search-ci",       "check-in search:",  "c" },
     { "search-doc",      "document search:",  "d" },
     { "search-tkt",      "ticket search:",    "t" },
     { "search-wiki",     "wiki search:",      "w" },
     { "search-technote", "tech note search:", "e" },
     { "search-forum",    "forum search:",     "f" },
  };







|
>
>
>






|
>
>
>
>







1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
**     stemmer (on|off)   Turn the Porter stemmer on or off for indexed
**                        search.  (Unindexed search is never stemmed.)
**
** The current search settings are displayed after any changes are applied.
** Run this command with no arguments to simply see the settings.
*/
void fts_config_cmd(void){
  static const struct { 
    int iCmd;
    const char *z;
  } aCmd[] = {
     { 1,  "reindex"  },
     { 2,  "index"    },
     { 3,  "disable"  },
     { 4,  "enable"   },
     { 5,  "stemmer"  },
  };
  static const struct {
    const char *zSetting;
    const char *zName;
    const char *zSw;
  } aSetng[] = {
     { "search-ci",       "check-in search:",  "c" },
     { "search-doc",      "document search:",  "d" },
     { "search-tkt",      "ticket search:",    "t" },
     { "search-wiki",     "wiki search:",      "w" },
     { "search-technote", "tech note search:", "e" },
     { "search-forum",    "forum search:",     "f" },
  };
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965


1966
1967
1968

1969
1970
1971
1972
1973
1974



1975
1976
1977
1978
1979
1980
1981
1982
1983










1984




1985
1986
1987
1988
1989
1990
1991
1992
1993

1994
1995








1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012

2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023

2024
2025




2026


2027





2028


2029
2030

2031
2032

2033
2034
2035
2036
*/
void search_data_page(void){
  Stmt q;
  const char *zId = P("id");
  const char *zType = P("y");
  const char *zIdxed = P("ixed");
  int id;
  int cnt = 0;
  login_check_credentials();
  if( !g.perm.Admin ){ login_needed(0); return; }
  if( !search_index_exists() ){
    @ <p>Indexed search is disabled
    style_footer();
    return;
  }


  if( zId!=0 && (id = atoi(zId))>0 ){
    /* Show information about a single ftsdocs entry */
    style_header("Information about ftsdoc entry %d", id);

    db_prepare(&q,
      "SELECT type||rid, name, idxed, label, url, datetime(mtime)"
      "  FROM ftsdocs WHERE rowid=%d", id
    );
    if( db_step(&q)==SQLITE_ROW ){
      const char *zUrl = db_column_text(&q,4);



      @ <table border=0>
      @ <tr><td align='right'>rowid:<td>&nbsp;&nbsp;<td>%d(id)
      @ <tr><td align='right'>id:<td><td>%s(db_column_text(&q,0))
      @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
      @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
      @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
      @ <tr><td align='right'>url:<td><td>
      @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
      @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))










      @ </table>




    }
    db_finalize(&q);
    style_footer();
    return;
  }
  if( zType!=0 && zType[0]!=0 && zType[1]==0 &&
      zIdxed!=0 && (zIdxed[0]=='1' || zIdxed[0]=='0') && zIdxed[1]==0
  ){
    int ixed = zIdxed[0]=='1';

    style_header("List of '%c' documents that are%s indexed",
                 zType[0], ixed ? "" : " not");








    db_prepare(&q,
      "SELECT rowid, type||rid ||' '|| coalesce(label,'')"
      "  FROM ftsdocs WHERE type='%c' AND %s idxed",
      zType[0], ixed ? "" : "NOT"
    );
    @ <ul>
    while( db_step(&q)==SQLITE_ROW ){
      @ <li> <a href='test-ftsdocs?id=%d(db_column_int(&q,0))'>
      @ %h(db_column_text(&q,1))</a>
    }
    @ </ul>
    db_finalize(&q);
    style_footer();
    return;
  }
  style_header("Summary of ftsdocs");
  db_prepare(&q,

     "SELECT type, idxed, count(*) FROM ftsdocs"
     " GROUP BY 1, 2 ORDER BY 3 DESC"
  );
  @ <table border=1 cellpadding=3 cellspacing=0>
  @ <thead>
  @ <tr><th>Type<th>Indexed?<th>Count<th>Link
  @ </thead>
  @ <tbody>
  while( db_step(&q)==SQLITE_ROW ){
    const char *zType = db_column_text(&q,0);
    int idxed = db_column_int(&q,1);

    int n = db_column_int(&q,2);
    @ <tr><td>%h(zType)<td>%d(idxed)




    @ <td>%d(n)


    @ <td><a href='test-ftsdocs?y=%s(zType)&ixed=%d(idxed)'>listing</a>





    @ </tr>


    cnt += n;
  }

  @ </tbody><tfooter>
  @ <tr><th>Total<th><th>%d(cnt)<th>

  @ </tfooter>
  @ </table>
  style_footer();
}







|







>
>



>






>
>
>

|
|






>
>
>
>
>
>
>
>
>
>

>
>
>
>









>


>
>
>
>
>
>
>
>

















>
|
|



|




|
>
|
|
>
>
>
>
|
>
>
|
>
>
>
>
>

>
>
|

>

|
>




1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
*/
void search_data_page(void){
  Stmt q;
  const char *zId = P("id");
  const char *zType = P("y");
  const char *zIdxed = P("ixed");
  int id;
  int cnt1 = 0, cnt2 = 0, cnt3 = 0;
  login_check_credentials();
  if( !g.perm.Admin ){ login_needed(0); return; }
  if( !search_index_exists() ){
    @ <p>Indexed search is disabled
    style_footer();
    return;
  }
  search_sql_setup(g.db);
  style_submenu_element("Setup","%R/srchsetup");
  if( zId!=0 && (id = atoi(zId))>0 ){
    /* Show information about a single ftsdocs entry */
    style_header("Information about ftsdoc entry %d", id);
    style_submenu_element("Summary","%R/test-ftsdocs");
    db_prepare(&q,
      "SELECT type||rid, name, idxed, label, url, datetime(mtime)"
      "  FROM ftsdocs WHERE rowid=%d", id
    );
    if( db_step(&q)==SQLITE_ROW ){
      const char *zUrl = db_column_text(&q,4);
      const char *zDocId = db_column_text(&q,0);
      char *zName;
      char *z;
      @ <table border=0>
      @ <tr><td align='right'>docid:<td>&nbsp;&nbsp;<td>%d(id)
      @ <tr><td align='right'>id:<td><td>%s(zDocId)
      @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
      @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
      @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
      @ <tr><td align='right'>url:<td><td>
      @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
      @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))
      z = db_text(0, "SELECT title FROM ftsidx WHERE docid=%d",id);
      if( z && z[0] ){
        @ <tr><td align="right">title:<td><td>%h(z)
        fossil_free(z);
      }
      z = db_text(0, "SELECT body FROM ftsidx WHERE docid=%d",id);
      if( z && z[0] ){
        @ <tr><td align="right" valign="top">body:<td><td>%h(z)
        fossil_free(z);
      }
      @ </table>
      zName = mprintf("Indexed '%c' docs",zDocId[0]);
      style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=1",zDocId[0]);
      zName = mprintf("Unindexed '%c' docs",zDocId[0]);
      style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=0",zDocId[0]);
    }
    db_finalize(&q);
    style_footer();
    return;
  }
  if( zType!=0 && zType[0]!=0 && zType[1]==0 &&
      zIdxed!=0 && (zIdxed[0]=='1' || zIdxed[0]=='0') && zIdxed[1]==0
  ){
    int ixed = zIdxed[0]=='1';
    char *zName;
    style_header("List of '%c' documents that are%s indexed",
                 zType[0], ixed ? "" : " not");
    style_submenu_element("Summary","%R/test-ftsdocs");
    if( ixed==0 ){
      zName = mprintf("Indexed '%c' docs",zType[0]);
      style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=1",zType[0]);
    }else{
      zName = mprintf("Unindexed '%c' docs",zType[0]);
      style_submenu_element(zName,"%R/test-ftsdocs?y=%c&ixed=0",zType[0]);
    }
    db_prepare(&q,
      "SELECT rowid, type||rid ||' '|| coalesce(label,'')"
      "  FROM ftsdocs WHERE type='%c' AND %s idxed",
      zType[0], ixed ? "" : "NOT"
    );
    @ <ul>
    while( db_step(&q)==SQLITE_ROW ){
      @ <li> <a href='test-ftsdocs?id=%d(db_column_int(&q,0))'>
      @ %h(db_column_text(&q,1))</a>
    }
    @ </ul>
    db_finalize(&q);
    style_footer();
    return;
  }
  style_header("Summary of ftsdocs");
  db_prepare(&q,
     "SELECT type, sum(idxed IS TRUE), sum(idxed IS FALSE), count(*)"
     "  FROM ftsdocs"
     " GROUP BY 1 ORDER BY 4 DESC"
  );
  @ <table border=1 cellpadding=3 cellspacing=0>
  @ <thead>
  @ <tr><th>Type<th>Indexed<th>Unindexed<th>Total
  @ </thead>
  @ <tbody>
  while( db_step(&q)==SQLITE_ROW ){
    const char *zType = db_column_text(&q,0);
    int nIndexed = db_column_int(&q, 1);
    int nUnindexed = db_column_int(&q, 2);
    int nTotal = db_column_int(&q, 3);
    @ <tr><td>%h(zType)
    if( nIndexed>0 ){
      @ <td align="right"><a href='%R/test-ftsdocs?y=%s(zType)&ixed=1'>\
      @ %d(nIndexed)</a>
    }else{
      @ <td align="right">0
    }
    if( nUnindexed>0 ){
      @ <td align="right"><a href='%R/test-ftsdocs?y=%s(zType)&ixed=0'>\
      @ %d(nUnindexed)</a>
    }else{
      @ <td align="right">0
    }
    @ <td align="right">%d(nTotal)
    @ </tr>
    cnt1 += nIndexed;
    cnt2 += nUnindexed;
    cnt3 += nTotal;
  }
  db_finalize(&q);
  @ </tbody><tfooter>
  @ <tr><th>Total<th align="right">%d(cnt1)<th align="right">%d(cnt2)
  @ <th align="right">%d(cnt3)
  @ </tfooter>
  @ </table>
  style_footer();
}
Changes to src/setup.c.
1638
1639
1640
1641
1642
1643
1644

1645
1646
1647
1648
1649
1650
1651
  }
  if( search_index_exists() ){
    @ <p>Currently using an SQLite FTS4 search index. This makes search
    @ run faster, especially on large repositories, but takes up space.</p>
    onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
    @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
    @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">

  }else{
    @ <p>The SQLite FTS4 search index is disabled.  All searching will be
    @ a full-text scan.  This usually works fine, but can be slow for
    @ larger repositories.</p>
    onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
    @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
  }







>







1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
  }
  if( search_index_exists() ){
    @ <p>Currently using an SQLite FTS4 search index. This makes search
    @ run faster, especially on large repositories, but takes up space.</p>
    onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
    @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
    @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
    style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
  }else{
    @ <p>The SQLite FTS4 search index is disabled.  All searching will be
    @ a full-text scan.  This usually works fine, but can be slow for
    @ larger repositories.</p>
    onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
    @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
  }