Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | If a ticket definition has a field named "mimetype" then use the specified mimetype when parsing ticket content to extract backlinks. Add the ability to extract backlinks from markdown-formatted text. Add the /test-backlinks webpage and the test-backlink command for debugging. |
|---|---|
| Downloads: | Tarball | ZIP archive |
| Timelines: | family | ancestors | descendants | both | backlink-updates |
| Files: | files | file ages | folders |
| SHA3-256: |
7c13a57358ae16a764900ac11d30aebf |
| User & Date: | drh 2020-04-16 16:52:07.408 |
References
|
2022-05-14
| ||
| 17:42 | Fix a subtle bug in <code>ticket_insert()</code> which may lead to redundant rows in the BACKLINK table. The bug appeared in [7c13a57358ae]. check-in: 3b42738e36 user: george tags: generated-tkt-mimetype | |
Context
|
2020-04-16
| ||
| 20:06 | An attempt to begin scanning wiki for backlinks. It does not currently work. I suspect a problem in the markdown link scanner. check-in: f0b0293ba7 user: drh tags: backlink-updates | |
| 16:52 | If a ticket definition has a field named "mimetype" then use the specified mimetype when parsing ticket content to extract backlinks. Add the ability to extract backlinks from markdown-formatted text. Add the /test-backlinks webpage and the test-backlink command for debugging. check-in: 7c13a57358 user: drh tags: backlink-updates | |
| 13:06 | Begin breaking out the code for BACKLINK processing into a separate source file: backlink.c check-in: 10c75204ef user: drh tags: backlink-updates | |
Changes
Changes to src/backlink.c.
| ︙ | ︙ | |||
54 55 56 57 58 59 60 |
db_prepare(&q, "%s", blob_sql_text(&sql));
www_print_timeline(&q, TIMELINE_DISJOINT|TIMELINE_GRAPH|TIMELINE_NOSCROLL,
0, 0, 0, 0, 0, 0);
db_finalize(&q);
}
/*
| | | 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
db_prepare(&q, "%s", blob_sql_text(&sql));
www_print_timeline(&q, TIMELINE_DISJOINT|TIMELINE_GRAPH|TIMELINE_NOSCROLL,
0, 0, 0, 0, 0, 0);
db_finalize(&q);
}
/*
** WEBPAGE: test-backlink-timeline
**
** Show a timeline of all check-ins and other events that have entries
** in the backlink table. This is used for testing the rendering
** of the "References" section of the /info page.
*/
void backlink_timeline_page(void){
Blob sql;
|
| ︙ | ︙ | |||
86 87 88 89 90 91 92 |
blob_append_sql(&sql, " AND event.objid IN ok ORDER BY mtime DESC");
db_prepare(&q, "%s", blob_sql_text(&sql));
www_print_timeline(&q, TIMELINE_DISJOINT|TIMELINE_GRAPH|TIMELINE_NOSCROLL,
0, 0, 0, 0, 0, 0);
db_finalize(&q);
style_footer();
}
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 |
blob_append_sql(&sql, " AND event.objid IN ok ORDER BY mtime DESC");
db_prepare(&q, "%s", blob_sql_text(&sql));
www_print_timeline(&q, TIMELINE_DISJOINT|TIMELINE_GRAPH|TIMELINE_NOSCROLL,
0, 0, 0, 0, 0, 0);
db_finalize(&q);
style_footer();
}
/*
** WEBPAGE: test-backlinks
**
** Show a table of all backlinks. Admin access only.
*/
void backlink_table_page(void){
Stmt q;
int n;
login_check_credentials();
if( !g.perm.Admin ){
login_needed(g.anon.Admin);
return;
}
style_header("Backlink Table (Internal Testing Use)");
n = db_int(0, "SELECT count(*) FROM backlink");
@ <p>%d(n) backlink table entries:</p>
db_prepare(&q,
"SELECT target, srctype, srcid, datetime(mtime) FROM backlink"
);
style_table_sorter();
@ <table border="1" cellpadding="2" cellspacing="0" \
@ class='sortable' data-column-types='ttt' data-init-sort='0'>
@ <thead><tr><th> Source <th> Target <th> mtime </tr></thead>
@ <tbody>
while( db_step(&q)==SQLITE_ROW ){
const char *zTarget = db_column_text(&q, 0);
int srctype = db_column_int(&q, 1);
int srcid = db_column_int(&q, 2);
const char *zMtime = db_column_text(&q, 3);
static const char *azSrcType[] = { "comment", "ticket", "wiki", "unknown" };
@ <tr><td><a href="%R/info/%h(zTarget)">%h(zTarget)</a>
switch( srctype ){
case BKLNK_COMMENT: {
@ <td><a href="%R/info?name=rid:%d(srcid)">comment-%d(srcid)</a>
break;
}
case BKLNK_TICKET: {
@ <td><a href="%R/info?name=rid:%d(srcid)">ticket-%d(srcid)</a>
break;
}
case BKLNK_WIKI: {
@ <td><a href="%R/info?name=rid:%d(srcid)">wiki-%d(srcid)</a>
break;
}
default: {
@ <td>unknown(%d(srctype)) - %d(srcid)
break;
}
}
@ <td>%h(zMtime)</tr>
}
@ </tbody>
@ </table>
db_finalize(&q);
style_footer();
}
/*
** Structure used to pass down state information through the
** markup formatters into the BACKLINK generator.
*/
#if INTERFACE
struct Backlink {
int srcid; /* srcid for the source document */
int srctype; /* One of BKLNK_*. 0=comment 1=ticket 2=wiki */
double mtime; /* mtime field for new BACKLINK table entries */
};
#endif
/*
** zTarget is a hyperlink target in some markup format. If this
** target is a self-reference to some other object in the repository,
** then create an appropriate backlink.
*/
void backlink_create(Backlink *p, const char *zTarget, int nTarget){
char zLink[HNAME_MAX+4];
if( zTarget==0 ) return;
if( nTarget<4 ) return;
if( nTarget>=10 && strncmp(zTarget,"/info/",6)==0 ){
zTarget += 6;
nTarget -= 6;
}
if( nTarget>HNAME_MAX ) return;
if( !validate16(zTarget, nTarget) ) return;
memcpy(zLink, zTarget, nTarget);
zLink[nTarget] = 0;
canonical16(zLink, nTarget);
db_multi_exec(
"REPLACE INTO backlink(target,srctype,srcid,mtime)"
"VALUES(%Q,%d,%d,%.17g)", zLink, p->srctype, p->srcid, p->mtime
);
}
/*
** This routine is called by the markdown formatter for each hyperlink.
** If the hyperlink is a backlink, add it to the BACKLINK table.
*/
static int backlink_md_link(
Blob *ob, /* Write output text here (not used in this case) */
Blob *target, /* The hyperlink target */
Blob *title, /* Hyperlink title */
Blob *content, /* Content of the link */
void *opaque
){
Backlink *p = (Backlink*)opaque;
char *zTarget = blob_buffer(target);
int nTarget = blob_size(target);
backlink_create(p, zTarget, nTarget);
return 1;
}
/*
** Scan markdown text and add self-hyperlinks to the BACKLINK table.
*/
void markdown_extract_links(
char *zInputText,
Backlink *p
){
struct mkd_renderer html_renderer = {
0, /* prolog */
0, /* epilog */
0, /* blockcode */
0, /* blockquote */
0, /* raw_block */
0, /* header */
0, /* hrule */
0, /* list */
0, /* list_item */
0, /* paragraph */
0, /* table */
0, /* table_cell */
0, /* table_row */
0, /* autolink */
0, /* code_span */
0, /* double-emphasis */
0, /* emphasis */
0, /* image */
0, /* line_break */
backlink_md_link, /* link */
0, /* raw_span */
0, /* triple_emphasis */
0, /* entity */
0, /* normal_text */
"*_", /* emphasis characters */
0 /* client data */
};
Blob out, in;
html_renderer.opaque = (void*)p;
blob_init(&out, 0, 0);
blob_init(&in, zInputText, -1);
markdown(&out, &in, &html_renderer);
blob_reset(&out);
blob_reset(&in);
}
/*
** Parse text looking for hyperlinks. Insert references into the
** BACKLINK table.
*/
void backlink_extract(
char *zSrc, /* Input text from which links are extracted */
const char *zMimetype, /* Mimetype of input. NULL means fossil-wiki */
int srcid, /* srcid for the source document */
int srctype, /* One of BKLNK_*. 0=comment 1=ticket 2=wiki */
double mtime, /* mtime field for new BACKLINK table entries */
int replaceFlag /* True to overwrite prior BACKLINK entries */
){
Backlink bklnk;
if( replaceFlag ){
db_multi_exec("DELETE FROM backlink WHERE srctype=%d AND srcid=%d",
srctype, srcid);
}
bklnk.srcid = srcid;
assert( srctype>=BKLNK_COMMENT && srctype<=BKLNK_WIKI );
bklnk.srctype = srctype;
bklnk.mtime = mtime;
if( zMimetype==0 || strstr(zMimetype,"wiki")!=0 ){
wiki_extract_links(zSrc, &bklnk, srctype==BKLNK_COMMENT ? WIKI_INLINE : 0);
}else if( strstr(zMimetype,"markdown")!=0 ){
markdown_extract_links(zSrc, &bklnk);
}
}
/*
** COMMAND: test-backlinks
**
** Usage: %fossil test-backlinks SRCID SRCTYPE ?OPTIONS? INPUT-FILE
**
** Read the content of INPUT-FILE and pass it into the backlink_extract()
** routine. But instead of adding backlinks to the backlink table,
** just print them on stdout. SRCID and SRCTYPE are integers.
**
** Options:
** --mtime DATETIME Use an alternative date/time. Defaults to the
** current date/time.
** --mimetype TYPE Use an alternative mimetype.
*/
void test_backlinks_cmd(void){
const char *zMTime = find_option("mtime",0,1);
const char *zMimetype = find_option("mimetype",0,1);
Blob in;
int srcid;
int srctype;
double mtime;
verify_all_options();
if( g.argc!=5 ){
usage("SRCTYPE SRCID INPUTFILE");
}
srctype = atoi(g.argv[2]);
if( srctype<0 || srctype>2 ){
fossil_fatal("SRCTYPE should be a integer 0, 1, or 2");
}
srcid = atoi(g.argv[3]);
blob_read_from_file(&in, g.argv[4], ExtFILE);
sqlite3_open(":memory:",&g.db);
if( zMTime==0 ) zMTime = "now";
mtime = db_double(1721059.5,"SELECT julianday(%Q)",zMTime);
g.fSqlPrint = 1;
sqlite3_create_function(g.db, "print", -1, SQLITE_UTF8, 0,db_sql_print,0,0);
db_multi_exec(
"CREATE TEMP TABLE backlink(target,srctype,srcid,mtime);\n"
"CREATE TRIGGER backlink_insert BEFORE INSERT ON backlink BEGIN\n"
" SELECT print("
" 'target='||quote(new.target)||"
" ' srctype='||quote(new.srctype)||"
" ' srcid='||quote(new.srcid)||"
" ' mtime='||datetime(new.mtime));\n"
" SELECT raise(ignore);\n"
"END;"
);
backlink_extract(blob_str(&in),zMimetype,srcid,srctype,mtime,0);
blob_reset(&in);
}
|
Changes to src/db.c.
| ︙ | ︙ | |||
2308 2309 2310 2311 2312 2313 2314 | /* ** SQL functions for debugging. ** ** The print() function writes its arguments on stdout, but only ** if the -sqlprint command-line option is turned on. */ | | | 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 |
/*
** SQL functions for debugging.
**
** The print() function writes its arguments on stdout, but only
** if the -sqlprint command-line option is turned on.
*/
void db_sql_print(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
int i;
if( g.fSqlPrint ){
for(i=0; i<argc; i++){
|
| ︙ | ︙ |
Changes to src/manifest.c.
| ︙ | ︙ | |||
2160 2161 2162 2163 2164 2165 2166 |
rid, p->zUser, p->zComment,
TAG_BGCOLOR, rid,
TAG_USER, rid,
TAG_COMMENT, rid, p->rDate
);
zCom = db_text(0, "SELECT coalesce(ecomment, comment) FROM event"
" WHERE rowid=last_insert_rowid()");
| | | 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 |
rid, p->zUser, p->zComment,
TAG_BGCOLOR, rid,
TAG_USER, rid,
TAG_COMMENT, rid, p->rDate
);
zCom = db_text(0, "SELECT coalesce(ecomment, comment) FROM event"
" WHERE rowid=last_insert_rowid()");
backlink_extract(zCom, 0, rid, BKLNK_COMMENT, p->rDate, 1);
fossil_free(zCom);
/* If this is a delta-manifest, record the fact that this repository
** contains delta manifests, to free the "commit" logic to generate
** new delta manifests.
*/
if( p->zBaseline!=0 ){
|
| ︙ | ︙ |
Changes to src/tag.c.
| ︙ | ︙ | |||
218 219 220 221 222 223 224 |
}
}
if( zCol ){
db_multi_exec("UPDATE event SET \"%w\"=%Q WHERE objid=%d",
zCol, zValue, rid);
if( tagid==TAG_COMMENT ){
char *zCopy = mprintf("%s", zValue);
| | | 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 |
}
}
if( zCol ){
db_multi_exec("UPDATE event SET \"%w\"=%Q WHERE objid=%d",
zCol, zValue, rid);
if( tagid==TAG_COMMENT ){
char *zCopy = mprintf("%s", zValue);
backlink_extract(zCopy, 0, rid, BKLNK_COMMENT, mtime, 1);
free(zCopy);
}
}
if( tagid==TAG_DATE ){
db_multi_exec("UPDATE event "
" SET mtime=julianday(%Q),"
" omtime=coalesce(omtime,mtime)"
|
| ︙ | ︙ |
Changes to src/tkt.c.
| ︙ | ︙ | |||
192 193 194 195 196 197 198 199 200 201 202 203 204 205 |
** Return the new rowid of the TICKET table entry.
*/
static int ticket_insert(const Manifest *p, int rid, int tktid){
Blob sql1, sql2, sql3;
Stmt q;
int i, j;
char *aUsed;
if( tktid==0 ){
db_multi_exec("INSERT INTO ticket(tkt_uuid, tkt_mtime) "
"VALUES(%Q, 0)", p->zTicketUuid);
tktid = db_last_insert_rowid();
}
blob_zero(&sql1);
| > | 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 |
** Return the new rowid of the TICKET table entry.
*/
static int ticket_insert(const Manifest *p, int rid, int tktid){
Blob sql1, sql2, sql3;
Stmt q;
int i, j;
char *aUsed;
const char *zMimetype = 0;
if( tktid==0 ){
db_multi_exec("INSERT INTO ticket(tkt_uuid, tkt_mtime) "
"VALUES(%Q, 0)", p->zTicketUuid);
tktid = db_last_insert_rowid();
}
blob_zero(&sql1);
|
| ︙ | ︙ | |||
231 232 233 234 235 236 237 |
const char *zUsedByName = zName;
if( zUsedByName[0]=='+' ){
zUsedByName++;
}
blob_append_sql(&sql2, ",\"%w\"", zUsedByName);
blob_append_sql(&sql3, ",%Q", p->aField[i].zValue);
}
| > > > > | > > > > > | | | 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 |
const char *zUsedByName = zName;
if( zUsedByName[0]=='+' ){
zUsedByName++;
}
blob_append_sql(&sql2, ",\"%w\"", zUsedByName);
blob_append_sql(&sql3, ",%Q", p->aField[i].zValue);
}
if( strcmp(zBaseName,"mimetype")==0 ){
zMimetype = p->aField[i].zValue;
}
}
if( rid>0 ){
for(i=0; i<p->nField; i++){
const char *zName = p->aField[i].zName;
const char *zBaseName = zName[0]=='+' ? zName+1 : zName;
j = fieldId(zBaseName);
if( j<0 ) continue;
backlink_extract(p->aField[i].zValue, zMimetype, rid, BKLNK_TICKET,
p->rDate, i==0);
}
}
blob_append_sql(&sql1, " WHERE tkt_id=%d", tktid);
db_prepare(&q, "%s", blob_sql_text(&sql1));
db_bind_double(&q, ":mtime", p->rDate);
db_step(&q);
db_finalize(&q);
|
| ︙ | ︙ |
Changes to src/wikiformat.c.
| ︙ | ︙ | |||
1867 1868 1869 1870 1871 1872 1873 | ** ** Where "target" can be either an artifact ID prefix or a wiki page ** name. For each such hyperlink found, add an entry to the ** backlink table. */ void wiki_extract_links( char *z, /* The wiki text from which to extract links */ | | < < < < < < < | < < < < | < < < < < < < | 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 |
**
** Where "target" can be either an artifact ID prefix or a wiki page
** name. For each such hyperlink found, add an entry to the
** backlink table.
*/
void wiki_extract_links(
char *z, /* The wiki text from which to extract links */
Backlink *pBklnk, /* Backlink extraction context */
int flags /* wiki parsing flags */
){
Renderer renderer;
int tokenType;
ParsedMarkup markup;
int n;
int inlineOnly;
int wikiHtmlOnly = 0;
memset(&renderer, 0, sizeof(renderer));
renderer.state = ALLOW_WIKI|AT_NEWLINE|AT_PARAGRAPH;
if( flags & WIKI_NOBLOCK ){
renderer.state |= INLINE_MARKUP_ONLY;
}
if( wikiUsesHtml() ){
renderer.state |= WIKI_HTMLONLY;
wikiHtmlOnly = 1;
}
inlineOnly = (renderer.state & INLINE_MARKUP_ONLY)!=0;
while( z[0] ){
if( wikiHtmlOnly ){
n = nextRawToken(z, &renderer, &tokenType);
}else{
n = nextWikiToken(z, &renderer, &tokenType);
}
switch( tokenType ){
case TOKEN_LINK: {
char *zTarget;
int i;
zTarget = &z[1];
for(i=0; zTarget[i] && zTarget[i]!='|' && zTarget[i]!=']'; i++){}
while(i>1 && zTarget[i-1]==' '){ i--; }
backlink_create(pBklnk, zTarget, i);
break;
}
case TOKEN_MARKUP: {
const char *zId;
int iDiv;
parseMarkup(&markup, z);
|
| ︙ | ︙ |