Fossil

Check-in [941280ae0a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Show the table of public phantoms directly on the security audit page. Dig deeper looking for the source of public phantoms.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 941280ae0aa53d1da49de9d20b6f6b3771039c7a6bf355ca97323c8c262a1aaa
User & Date: drh 2020-04-14 14:27:40.338
References
2020-04-17
00:14 Wiki page "checkin/b4beadb5078a03421a62f5a205cea1a52febe84b83c5189c9d271cd54a459136" artifact: 34231d3a62 user: drh
Context
2020-04-14
21:38
Typo fix in the history.md document. check-in: 30b0b112b2 user: drh tags: trunk
14:27
Show the table of public phantoms directly on the security audit page. Dig deeper looking for the source of public phantoms. check-in: 941280ae0a user: drh tags: trunk
13:32
Add the /phantoms webpage that lists public phantom artifacts. Check the number of public phantom artifacts and puts a warning if the number is non-zero on the Security Audit page. check-in: 83db244395 user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/name.c.
966
967
968
969
970
971
972

































































973
974
975
976
977
978
979
@   type TEXT,                     -- file, checkin, wiki, ticket, etc.
@   rcvid INT,                     -- When the artifact was received
@   summary TEXT,                  -- Summary comment for the object
@   ref TEXT                       -- hash of an object to link against
@ );
@ CREATE INDEX desctype ON description(summary) WHERE summary='unknown';
;


































































/*
** Create the description table if it does not already exists.
** Populate fields of this table with descriptions for all artifacts
** whose RID matches the SQL expression in zWhere.
*/
void describe_artifacts(const char *zWhere){







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
@   type TEXT,                     -- file, checkin, wiki, ticket, etc.
@   rcvid INT,                     -- When the artifact was received
@   summary TEXT,                  -- Summary comment for the object
@   ref TEXT                       -- hash of an object to link against
@ );
@ CREATE INDEX desctype ON description(summary) WHERE summary='unknown';
;

/*
** Attempt to describe all phantom artifacts.  The artifacts are
** already loaded into the description table and have summary='unknown'.
** This routine attempts to generate a better summary, and possibly
** fill in the ref field.
*/
static void describe_unknown_artifacts(){
  /* Try to figure out the origin of unknown artifacts */
  db_multi_exec(
    "REPLACE INTO description(rid,uuid,isPrivate,type,summary,ref)\n"
    "  SELECT description.rid, description.uuid, isPrivate, type,\n"
    "         CASE WHEN plink.isprim THEN '' ELSE 'merge ' END ||\n"
    "         'parent of check-in', blob.uuid\n"
    "    FROM description, plink, blob\n"
    "   WHERE description.summary='unknown'\n"
    "     AND plink.pid=description.rid\n"
    "     AND blob.rid=plink.cid;"
  );
  db_multi_exec(
    "REPLACE INTO description(rid,uuid,isPrivate,type,summary,ref)\n"
    "  SELECT description.rid, description.uuid, isPrivate, type,\n"
    "         'child of check-in', blob.uuid\n"
    "    FROM description, plink, blob\n"
    "   WHERE description.summary='unknown'\n"
    "     AND plink.cid=description.rid\n"
    "     AND blob.rid=plink.pid;"
  );
  db_multi_exec(
    "REPLACE INTO description(rid,uuid,isPrivate,type,summary,ref)\n"
    "  SELECT description.rid, description.uuid, isPrivate, type,\n"
    "         'check-in referenced by \"'||tag.tagname ||'\" tag',\n"
    "         blob.uuid\n"
    "    FROM description, tagxref, tag, blob\n"
    "   WHERE description.summary='unknown'\n"
    "     AND tagxref.origid=description.rid\n"
    "     AND tag.tagid=tagxref.tagid\n"
    "     AND blob.rid=tagxref.srcid;"
  );
  db_multi_exec(
    "REPLACE INTO description(rid,uuid,isPrivate,type,summary,ref)\n"
    "  SELECT description.rid, description.uuid, isPrivate, type,\n"
    "         'file \"'||filename.name||'\"',\n"
    "         blob.uuid\n"
    "    FROM description, mlink, filename, blob\n"
    "   WHERE description.summary='unknown'\n"
    "     AND mlink.fid=description.rid\n"
    "     AND blob.rid=mlink.mid\n"
    "     AND filename.fnid=mlink.fnid;"
  );
  if( !db_exists("SELECT 1 FROM description WHERE summary='unknown'") ){
    return;
  }
  add_content_sql_commands(g.db);
  db_multi_exec(
    "REPLACE INTO description(rid,uuid,isPrivate,type,summary,ref)\n"
    "  SELECT description.rid, description.uuid, isPrivate, type,\n"
    "         'referenced by cluster', blob.uuid\n"
    "    FROM description, tagxref, blob\n"
    "   WHERE description.summary='unknown'\n"
    "     AND tagxref.tagid=(SELECT tagid FROM tag WHERE tagname='cluster')\n"
    "     AND blob.rid=tagxref.rid\n"
    "     AND content(blob.uuid) GLOB ('*M '||blob.uuid||'*');"
  );
}

/*
** Create the description table if it does not already exists.
** Populate fields of this table with descriptions for all artifacts
** whose RID matches the SQL expression in zWhere.
*/
void describe_artifacts(const char *zWhere){
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146

1147
1148
1149
1150
1151
1152
1153
  );

  /* Mark private elements */
  db_multi_exec(
   "UPDATE description SET isPrivate=1 WHERE rid IN private"
  );

  /* Try to figure out the origin of unknown artifacts */
  db_multi_exec(
    "REPLACE INTO description(rid,uuid,isPrivate,type,summary,ref)\n"
    "  SELECT description.rid, description.uuid, isPrivate, type,\n"
    "         CASE WHEN plink.isprim THEN '' ELSE 'merge ' END ||\n"
    "         'parent of check-in', blob.uuid\n"
    "    FROM description, plink, blob\n"
    "   WHERE description.summary='unknown'\n"
    "     AND plink.pid=description.rid\n"
    "     AND blob.rid=plink.cid;"
  );
  db_multi_exec(
    "REPLACE INTO description(rid,uuid,isPrivate,type,summary,ref)\n"
    "  SELECT description.rid, description.uuid, isPrivate, type,\n"
    "         'check-in referenced by \"'||tag.tagname ||'\" tag',\n"
    "         blob.uuid\n"
    "    FROM description, tagxref, tag, blob\n"
    "   WHERE description.summary='unknown'\n"
    "     AND tagxref.origid=description.rid\n"
    "     AND tag.tagid=tagxref.tagid\n"
    "     AND blob.rid=tagxref.srcid;"
  );

}

/*
** Print the content of the description table on stdout.
**
** The description table is computed using the WHERE clause zWhere if
** the zWhere parameter is not NULL.  If zWhere is NULL, then this







<
<
<
<
<
<
|
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
>







1183
1184
1185
1186
1187
1188
1189






1190
1191














1192
1193
1194
1195
1196
1197
1198
1199
  );

  /* Mark private elements */
  db_multi_exec(
   "UPDATE description SET isPrivate=1 WHERE rid IN private"
  );







  if( db_exists("SELECT 1 FROM description WHERE summary='unknown'") ){
    describe_unknown_artifacts();














  }
}

/*
** Print the content of the description table on stdout.
**
** The description table is computed using the WHERE clause zWhere if
** the zWhere parameter is not NULL.  If zWhere is NULL, then this
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
  }
  @ </table>
  db_finalize(&q);
  style_footer();
}

/*
** WEBPAGE: phantoms
**
** Show a list of all "phantom" artifacts that are not marked as "private".
**
** A "phantom" artifact is an artifact whose hash named appears in some
** artifact but whose content is unknown.  For example, if a manifest
** references a particular SHA3 hash of a file, but that SHA3 hash is
** not on the shunning list and is not in the database, then the file
** is a phantom.  We know it exists, but we do not know its content.
**
** Whenever a sync occurs, both each party looks at its phantom list
** and for every phantom that is not also marked private, it asks the
** other party to send it the content.  This mechanism helps keep all
** repositories synced up.
**
** This page is similar to the /bloblist page in that it lists artifacts.
** But this page is a special case in that it only shows phantoms that
** are not private.  In other words, this page shows all phantoms that
** generate extra network traffic on every sync request.
*/
void phantom_list_page(void){
  Stmt q;
  char *zRange;

  login_check_credentials();
  if( !g.perm.Read ){ login_needed(g.anon.Read); return; }
  style_header("Public Phantom Artifacts");
  if( g.perm.Admin ){
    style_submenu_element("Artifact Log", "rcvfromlist");
    style_submenu_element("Artifact List", "bloblist");
  }
  if( g.perm.Write ){
    style_submenu_element("Artifact Stats", "artifact_stats");
  }
  zRange = mprintf("IN (SELECT rid FROM phantom EXCEPT"
                   " SELECT rid FROM private)");
  describe_artifacts(zRange);
  fossil_free(zRange);
  db_prepare(&q,
    "SELECT rid, uuid, summary, ref"
    "  FROM description ORDER BY rid"







|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<

|


<
<
<
<
<
<
<
<
<
<
<







1376
1377
1378
1379
1380
1381
1382
1383


















1384
1385
1386
1387











1388
1389
1390
1391
1392
1393
1394
  }
  @ </table>
  db_finalize(&q);
  style_footer();
}

/*
** Output HTML that shows a table of all public phantoms.


















*/
void table_of_public_phantoms(void){
  Stmt q;
  char *zRange;











  zRange = mprintf("IN (SELECT rid FROM phantom EXCEPT"
                   " SELECT rid FROM private)");
  describe_artifacts(zRange);
  fossil_free(zRange);
  db_prepare(&q,
    "SELECT rid, uuid, summary, ref"
    "  FROM description ORDER BY rid"
1390
1391
1392
1393
1394
1395
1396



































1397
1398
1399
1400
1401
1402
1403
    }else{
      @ <td>&nbsp;
    }
    @ </tr>
  }
  @ </table>
  db_finalize(&q);



































  style_footer();
}

/*
** WEBPAGE: bigbloblist
**
** Return a page showing the largest artifacts in the repository in order







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
    }else{
      @ <td>&nbsp;
    }
    @ </tr>
  }
  @ </table>
  db_finalize(&q);
}

/*
** WEBPAGE: phantoms
**
** Show a list of all "phantom" artifacts that are not marked as "private".
**
** A "phantom" artifact is an artifact whose hash named appears in some
** artifact but whose content is unknown.  For example, if a manifest
** references a particular SHA3 hash of a file, but that SHA3 hash is
** not on the shunning list and is not in the database, then the file
** is a phantom.  We know it exists, but we do not know its content.
**
** Whenever a sync occurs, both each party looks at its phantom list
** and for every phantom that is not also marked private, it asks the
** other party to send it the content.  This mechanism helps keep all
** repositories synced up.
**
** This page is similar to the /bloblist page in that it lists artifacts.
** But this page is a special case in that it only shows phantoms that
** are not private.  In other words, this page shows all phantoms that
** generate extra network traffic on every sync request.
*/
void phantom_list_page(void){
  login_check_credentials();
  if( !g.perm.Read ){ login_needed(g.anon.Read); return; }
  style_header("Public Phantom Artifacts");
  if( g.perm.Admin ){
    style_submenu_element("Artifact Log", "rcvfromlist");
    style_submenu_element("Artifact List", "bloblist");
  }
  if( g.perm.Write ){
    style_submenu_element("Artifact Stats", "artifact_stats");
  }
  table_of_public_phantoms();
  style_footer();
}

/*
** WEBPAGE: bigbloblist
**
** Return a page showing the largest artifacts in the repository in order
Changes to src/security_audit.c.
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574


575
576
577
578
579
580
581
    @ <li><p> Email alerts are disabled
  }

  n = db_int(0,"SELECT count(*) FROM ("
               "SELECT rid FROM phantom EXCEPT SELECT rid FROM private)");
  if( n>0 ){
    @ <li><p>\
    if( n==1 ){
      @ There is 1 public phantom artifact
    }else{
      @ There are %d(n) public phantom artifacts
    }
    @ (<a href="%R/phantoms">details</a>).
    @ Phantom artifacts are artifacts whose hash name is referenced by some
    @ other artifact but whose content is unknown.  Some phantoms are marked
    @ private and those are ignored.  But public phantoms cause unnecessary
    @ sync traffic and might represent malicious attempts to corrupt the
    @ repository structure.
    @ </p></li>


  }

  @ </ol>
  style_footer();
}

/*







<
|
<
<
<
<





|
>
>







556
557
558
559
560
561
562

563




564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
    @ <li><p> Email alerts are disabled
  }

  n = db_int(0,"SELECT count(*) FROM ("
               "SELECT rid FROM phantom EXCEPT SELECT rid FROM private)");
  if( n>0 ){
    @ <li><p>\

    @ There exists public phantom artifacts in this repository, shown below.




    @ Phantom artifacts are artifacts whose hash name is referenced by some
    @ other artifact but whose content is unknown.  Some phantoms are marked
    @ private and those are ignored.  But public phantoms cause unnecessary
    @ sync traffic and might represent malicious attempts to corrupt the
    @ repository structure.
    @ </p>
    table_of_public_phantoms();
    @ </li>
  }

  @ </ol>
  style_footer();
}

/*