Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Optimize sync of long cluster chains by sending all cluster artifacts on the third client/server round-trip. |
|---|---|
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA3-256: |
637fcef83023bad32ecf73db8cd9e55a |
| User & Date: | drh 2024-12-20 21:16:30.300 |
Context
|
2024-12-21
| ||
| 00:43 | Documentation updates due to today's changes. check-in: 5f8721753c user: drh tags: trunk | |
|
2024-12-20
| ||
| 21:16 | Optimize sync of long cluster chains by sending all cluster artifacts on the third client/server round-trip. check-in: 637fcef830 user: drh tags: trunk | |
| 19:50 | Further improvements to /clusterlist and to /info for a cluster. check-in: 9953b16976 user: drh tags: trunk | |
Changes
Changes to src/xfer.c.
| ︙ | ︙ | |||
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 |
pXfer->resync = db_column_int(&q, 1)-1;
}
}
db_finalize(&q);
if( cnt==0 ) pXfer->resync = 0;
return cnt;
}
/*
** Send an igot message for every artifact.
*/
static void send_all(Xfer *pXfer){
Stmt q;
db_prepare(&q,
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 |
pXfer->resync = db_column_int(&q, 1)-1;
}
}
db_finalize(&q);
if( cnt==0 ) pXfer->resync = 0;
return cnt;
}
/*
** Send an igot message for every cluster artifact that is not a phantom,
** is not shunned, is not private, and that is not in the UNCLUSTERED table.
** Return the number of cards sent.
*/
static int send_all_clusters(Xfer *pXfer){
Stmt q;
int cnt = 0;
const char *zExtra;
if( db_table_exists("temp","onremote") ){
zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
}else{
zExtra = "";
}
db_prepare(&q,
"SELECT uuid"
" FROM tagxref JOIN blob ON tagxref.rid=blob.rid AND tagxref.tagid=%d"
" WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
" AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
" AND NOT EXISTS(SELECT 1 FROM unclustered WHERE rid=blob.rid)"
" AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
TAG_CLUSTER, zExtra /*safe-for-%s*/
);
while( db_step(&q)==SQLITE_ROW ){
if( cnt==0 ) blob_appendf(pXfer->pOut, "# sending-clusters\n");
blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
cnt++;
}
db_finalize(&q);
if( cnt ) blob_appendf(pXfer->pOut, "# end-of-clusters\n");
return cnt;
}
/*
** Send an igot message for every artifact.
*/
static void send_all(Xfer *pXfer){
Stmt q;
db_prepare(&q,
|
| ︙ | ︙ | |||
1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 |
/* pragma req-links
**
** The client sends this message to the server to ask the server
** to tell it about alternative repositories in the reply.
*/
if( blob_eq(&xfer.aToken[1], "req-links") ){
bSendLinks = 1;
}
}else
/* Unknown message
*/
{
| > > > > > > > > > | 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 |
/* pragma req-links
**
** The client sends this message to the server to ask the server
** to tell it about alternative repositories in the reply.
*/
if( blob_eq(&xfer.aToken[1], "req-links") ){
bSendLinks = 1;
}else
/* pragma req-clusters
**
** This pragma requests that the server send igot cards for every
** cluster artifact that it knows about.
*/
if( blob_eq(&xfer.aToken[1], "req-clusters") ){
send_all_clusters(&xfer);
}
}else
/* Unknown message
*/
{
|
| ︙ | ︙ | |||
1979 1980 1981 1982 1983 1984 1985 |
){
int go = 1; /* Loop until zero */
int nCardSent = 0; /* Number of cards sent */
int nCardRcvd = 0; /* Number of cards received */
int nCycle = 0; /* Number of round trips to the server */
int size; /* Size of a config value or uvfile */
int origConfigRcvMask; /* Original value of configRcvMask */
| | | 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 |
){
int go = 1; /* Loop until zero */
int nCardSent = 0; /* Number of cards sent */
int nCardRcvd = 0; /* Number of cards received */
int nCycle = 0; /* Number of round trips to the server */
int size; /* Size of a config value or uvfile */
int origConfigRcvMask; /* Original value of configRcvMask */
int nFileRecv = 0; /* Number of files received */
int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
const char *zCookie; /* Server cookie */
i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
i64 nSent, nRcvd; /* Bytes sent and received (after compression) */
int cloneSeqno = 1; /* Sequence number for clones */
Blob send; /* Text we are sending to the server */
Blob recv; /* Reply we got back from the server */
|
| ︙ | ︙ | |||
2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 | const char *zOpType = 0;/* Push, Pull, Sync, Clone */ double rSkew = 0.0; /* Maximum time skew */ int uvHashSent = 0; /* The "pragma uv-hash" message has been sent */ int uvDoPush = 0; /* Generate uvfile messages to send to server */ int uvPullOnly = 0; /* 1: pull-only. 2: pull-only warning issued */ int nUvGimmeSent = 0; /* Number of uvgimme cards sent on this cycle */ int nUvFileRcvd = 0; /* Number of uvfile cards received on this cycle */ sqlite3_int64 mtime; /* Modification time on a UV file */ int autopushFailed = 0; /* Autopush following commit failed if true */ const char *zCkinLock; /* Name of check-in to lock. NULL for none */ const char *zClientId; /* A unique identifier for this check-out */ unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */ const int bOutIsTty = fossil_isatty(fossil_fileno(stdout)); | > | 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 | const char *zOpType = 0;/* Push, Pull, Sync, Clone */ double rSkew = 0.0; /* Maximum time skew */ int uvHashSent = 0; /* The "pragma uv-hash" message has been sent */ int uvDoPush = 0; /* Generate uvfile messages to send to server */ int uvPullOnly = 0; /* 1: pull-only. 2: pull-only warning issued */ int nUvGimmeSent = 0; /* Number of uvgimme cards sent on this cycle */ int nUvFileRcvd = 0; /* Number of uvfile cards received on this cycle */ int nGimmeRcvd = 0; /* Number of gimme cards recevied on the prev cycle */ sqlite3_int64 mtime; /* Modification time on a UV file */ int autopushFailed = 0; /* Autopush following commit failed if true */ const char *zCkinLock; /* Name of check-in to lock. NULL for none */ const char *zClientId; /* A unique identifier for this check-out */ unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */ const int bOutIsTty = fossil_isatty(fossil_fileno(stdout)); |
| ︙ | ︙ | |||
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 |
/* Client sends gimme cards for phantoms
*/
if( (syncFlags & SYNC_PULL)!=0
|| ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
){
request_phantoms(&xfer, mxPhantomReq);
}
if( syncFlags & SYNC_PUSH ){
send_unsent(&xfer);
nCardSent += send_unclustered(&xfer);
if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
}
/* Client sends configuration parameter requests. On a clone, delay sending
** this until the second cycle since the login card might fail on
** the first cycle.
*/
if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 || nCycle>0) ){
| > > > > > > | 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 |
/* Client sends gimme cards for phantoms
*/
if( (syncFlags & SYNC_PULL)!=0
|| ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
){
request_phantoms(&xfer, mxPhantomReq);
if( xfer.nGimmeSent>0 && nCycle==2 && (syncFlags & SYNC_PULL)!=0 ){
blob_appendf(&send, "pragma req-clusters\n");
}
}
if( syncFlags & SYNC_PUSH ){
send_unsent(&xfer);
nCardSent += send_unclustered(&xfer);
if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
if( nGimmeRcvd>0 && nCycle==2 ){
send_all_clusters(&xfer);
}
}
/* Client sends configuration parameter requests. On a clone, delay sending
** this until the second cycle since the login card might fail on
** the first cycle.
*/
if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 || nCycle>0) ){
|
| ︙ | ︙ | |||
2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 |
if( syncFlags & SYNC_PUSH ){
blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
nCardSent++;
}
go = 0;
nUvGimmeSent = 0;
nUvFileRcvd = 0;
nPriorArtifact = nArtifactRcvd;
/* Process the reply that came back from the server */
while( blob_line(&recv, &xfer.line) ){
if( blob_buffer(&xfer.line)[0]=='#' ){
const char *zLine = blob_buffer(&xfer.line);
if( memcmp(zLine, "# timestamp ", 12)==0 ){
| > | 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 |
if( syncFlags & SYNC_PUSH ){
blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
nCardSent++;
}
go = 0;
nUvGimmeSent = 0;
nUvFileRcvd = 0;
nGimmeRcvd = 0;
nPriorArtifact = nArtifactRcvd;
/* Process the reply that came back from the server */
while( blob_line(&recv, &xfer.line) ){
if( blob_buffer(&xfer.line)[0]=='#' ){
const char *zLine = blob_buffer(&xfer.line);
if( memcmp(zLine, "# timestamp ", 12)==0 ){
|
| ︙ | ︙ | |||
2447 2448 2449 2450 2451 2452 2453 |
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_hname(&xfer.aToken[1])
){
remote_unk(&xfer.aToken[1]);
if( syncFlags & SYNC_PUSH ){
int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
| > | > > | 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 |
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_hname(&xfer.aToken[1])
){
remote_unk(&xfer.aToken[1]);
if( syncFlags & SYNC_PUSH ){
int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
if( rid ){
send_file(&xfer, rid, &xfer.aToken[1], 0);
nGimmeRcvd++;
}
}
}else
/* igot HASH ?PRIVATEFLAG?
**
** Server announces that it has a particular file. If this is
** not a file that we have and we are pulling, then create a
|
| ︙ | ︙ |