Check-in [637fcef830]
Not logged in

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Optimize sync of long cluster chains by sending all cluster artifacts on the third client/server round-trip.
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 637fcef83023bad32ecf73db8cd9e55a122649e1373684d7aef0263b202541fc
User & Date: drh 2024-12-20 21:16:30.300
Context
2024-12-21
00:43
Documentation updates due to today's changes. check-in: 5f8721753c user: drh tags: trunk
2024-12-20
21:16
Optimize sync of long cluster chains by sending all cluster artifacts on the third client/server round-trip. check-in: 637fcef830 user: drh tags: trunk
19:50
Further improvements to /clusterlist and to /info for a cluster. check-in: 9953b16976 user: drh tags: trunk
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/xfer.c.
1039
1040
1041
1042
1043
1044
1045

































1046
1047
1048
1049
1050
1051
1052
      pXfer->resync = db_column_int(&q, 1)-1;
    }
  }
  db_finalize(&q);
  if( cnt==0 ) pXfer->resync = 0;
  return cnt;
}


































/*
** Send an igot message for every artifact.
*/
static void send_all(Xfer *pXfer){
  Stmt q;
  db_prepare(&q,







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
      pXfer->resync = db_column_int(&q, 1)-1;
    }
  }
  db_finalize(&q);
  if( cnt==0 ) pXfer->resync = 0;
  return cnt;
}

/*
** Send an igot message for every cluster artifact that is not a phantom,
** is not shunned, is not private, and that is not in the UNCLUSTERED table.
** Return the number of cards sent.
*/
static int send_all_clusters(Xfer *pXfer){
  Stmt q;
  int cnt = 0;
  const char *zExtra;
  if( db_table_exists("temp","onremote") ){
    zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
  }else{
    zExtra = "";
  }
  db_prepare(&q,
    "SELECT uuid"
    "  FROM tagxref JOIN blob ON tagxref.rid=blob.rid AND tagxref.tagid=%d"
    " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
    "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
    "   AND NOT EXISTS(SELECT 1 FROM unclustered WHERE rid=blob.rid)"
    "   AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
    TAG_CLUSTER, zExtra /*safe-for-%s*/
  );
  while( db_step(&q)==SQLITE_ROW ){
    if( cnt==0 ) blob_appendf(pXfer->pOut, "# sending-clusters\n");
    blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
    cnt++;
  }
  db_finalize(&q);
  if( cnt ) blob_appendf(pXfer->pOut, "# end-of-clusters\n");
  return cnt;
}

/*
** Send an igot message for every artifact.
*/
static void send_all(Xfer *pXfer){
  Stmt q;
  db_prepare(&q,
1779
1780
1781
1782
1783
1784
1785









1786
1787
1788
1789
1790
1791
1792
      /*    pragma req-links
      **
      ** The client sends this message to the server to ask the server
      ** to tell it about alternative repositories  in the reply.
      */
      if( blob_eq(&xfer.aToken[1], "req-links") ){
        bSendLinks = 1;









      }

    }else

    /* Unknown message
    */
    {







>
>
>
>
>
>
>
>
>







1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
      /*    pragma req-links
      **
      ** The client sends this message to the server to ask the server
      ** to tell it about alternative repositories  in the reply.
      */
      if( blob_eq(&xfer.aToken[1], "req-links") ){
        bSendLinks = 1;
      }else

      /*   pragma req-clusters
      **
      ** This pragma requests that the server send igot cards for every
      ** cluster artifact that it knows about.
      */
      if( blob_eq(&xfer.aToken[1], "req-clusters") ){
        send_all_clusters(&xfer);
      }

    }else

    /* Unknown message
    */
    {
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
){
  int go = 1;             /* Loop until zero */
  int nCardSent = 0;      /* Number of cards sent */
  int nCardRcvd = 0;      /* Number of cards received */
  int nCycle = 0;         /* Number of round trips to the server */
  int size;               /* Size of a config value or uvfile */
  int origConfigRcvMask;  /* Original value of configRcvMask */
  int nFileRecv;          /* Number of files received */
  int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
  const char *zCookie;    /* Server cookie */
  i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
  i64 nSent, nRcvd;       /* Bytes sent and received (after compression) */
  int cloneSeqno = 1;     /* Sequence number for clones */
  Blob send;              /* Text we are sending to the server */
  Blob recv;              /* Reply we got back from the server */







|







2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
){
  int go = 1;             /* Loop until zero */
  int nCardSent = 0;      /* Number of cards sent */
  int nCardRcvd = 0;      /* Number of cards received */
  int nCycle = 0;         /* Number of round trips to the server */
  int size;               /* Size of a config value or uvfile */
  int origConfigRcvMask;  /* Original value of configRcvMask */
  int nFileRecv = 0;      /* Number of files received */
  int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
  const char *zCookie;    /* Server cookie */
  i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
  i64 nSent, nRcvd;       /* Bytes sent and received (after compression) */
  int cloneSeqno = 1;     /* Sequence number for clones */
  Blob send;              /* Text we are sending to the server */
  Blob recv;              /* Reply we got back from the server */
2005
2006
2007
2008
2009
2010
2011

2012
2013
2014
2015
2016
2017
2018
  const char *zOpType = 0;/* Push, Pull, Sync, Clone */
  double rSkew = 0.0;     /* Maximum time skew */
  int uvHashSent = 0;     /* The "pragma uv-hash" message has been sent */
  int uvDoPush = 0;       /* Generate uvfile messages to send to server */
  int uvPullOnly = 0;     /* 1: pull-only.  2: pull-only warning issued */
  int nUvGimmeSent = 0;   /* Number of uvgimme cards sent on this cycle */
  int nUvFileRcvd = 0;    /* Number of uvfile cards received on this cycle */

  sqlite3_int64 mtime;    /* Modification time on a UV file */
  int autopushFailed = 0; /* Autopush following commit failed if true */
  const char *zCkinLock;  /* Name of check-in to lock.  NULL for none */
  const char *zClientId;  /* A unique identifier for this check-out */
  unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */
  const int bOutIsTty = fossil_isatty(fossil_fileno(stdout));








>







2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
  const char *zOpType = 0;/* Push, Pull, Sync, Clone */
  double rSkew = 0.0;     /* Maximum time skew */
  int uvHashSent = 0;     /* The "pragma uv-hash" message has been sent */
  int uvDoPush = 0;       /* Generate uvfile messages to send to server */
  int uvPullOnly = 0;     /* 1: pull-only.  2: pull-only warning issued */
  int nUvGimmeSent = 0;   /* Number of uvgimme cards sent on this cycle */
  int nUvFileRcvd = 0;    /* Number of uvfile cards received on this cycle */
  int nGimmeRcvd = 0;     /* Number of gimme cards recevied on the prev cycle */
  sqlite3_int64 mtime;    /* Modification time on a UV file */
  int autopushFailed = 0; /* Autopush following commit failed if true */
  const char *zCkinLock;  /* Name of check-in to lock.  NULL for none */
  const char *zClientId;  /* A unique identifier for this check-out */
  unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */
  const int bOutIsTty = fossil_isatty(fossil_fileno(stdout));

2179
2180
2181
2182
2183
2184
2185



2186
2187
2188
2189
2190



2191
2192
2193
2194
2195
2196
2197

    /* Client sends gimme cards for phantoms
    */
    if( (syncFlags & SYNC_PULL)!=0
     || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
    ){
      request_phantoms(&xfer, mxPhantomReq);



    }
    if( syncFlags & SYNC_PUSH ){
      send_unsent(&xfer);
      nCardSent += send_unclustered(&xfer);
      if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);



    }

    /* Client sends configuration parameter requests.  On a clone, delay sending
    ** this until the second cycle since the login card might fail on
    ** the first cycle.
    */
    if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 || nCycle>0) ){







>
>
>





>
>
>







2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246

    /* Client sends gimme cards for phantoms
    */
    if( (syncFlags & SYNC_PULL)!=0
     || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
    ){
      request_phantoms(&xfer, mxPhantomReq);
      if( xfer.nGimmeSent>0 && nCycle==2 && (syncFlags & SYNC_PULL)!=0 ){
        blob_appendf(&send, "pragma req-clusters\n");
      }
    }
    if( syncFlags & SYNC_PUSH ){
      send_unsent(&xfer);
      nCardSent += send_unclustered(&xfer);
      if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
      if( nGimmeRcvd>0 && nCycle==2 ){
        send_all_clusters(&xfer);
      }
    }

    /* Client sends configuration parameter requests.  On a clone, delay sending
    ** this until the second cycle since the login card might fail on
    ** the first cycle.
    */
    if( configRcvMask && ((syncFlags & SYNC_CLONE)==0 || nCycle>0) ){
2367
2368
2369
2370
2371
2372
2373

2374
2375
2376
2377
2378
2379
2380
    if( syncFlags & SYNC_PUSH ){
      blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
      nCardSent++;
    }
    go = 0;
    nUvGimmeSent = 0;
    nUvFileRcvd = 0;

    nPriorArtifact = nArtifactRcvd;

    /* Process the reply that came back from the server */
    while( blob_line(&recv, &xfer.line) ){
      if( blob_buffer(&xfer.line)[0]=='#' ){
        const char *zLine = blob_buffer(&xfer.line);
        if( memcmp(zLine, "# timestamp ", 12)==0 ){







>







2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
    if( syncFlags & SYNC_PUSH ){
      blob_appendf(&send, "push %s %s\n", zSCode, zPCode);
      nCardSent++;
    }
    go = 0;
    nUvGimmeSent = 0;
    nUvFileRcvd = 0;
    nGimmeRcvd = 0;
    nPriorArtifact = nArtifactRcvd;

    /* Process the reply that came back from the server */
    while( blob_line(&recv, &xfer.line) ){
      if( blob_buffer(&xfer.line)[0]=='#' ){
        const char *zLine = blob_buffer(&xfer.line);
        if( memcmp(zLine, "# timestamp ", 12)==0 ){
2447
2448
2449
2450
2451
2452
2453

2454


2455
2456
2457
2458
2459
2460
2461
      if( blob_eq(&xfer.aToken[0], "gimme")
       && xfer.nToken==2
       && blob_is_hname(&xfer.aToken[1])
      ){
        remote_unk(&xfer.aToken[1]);
        if( syncFlags & SYNC_PUSH ){
          int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);

          if( rid ) send_file(&xfer, rid, &xfer.aToken[1], 0);


        }
      }else

      /*   igot HASH  ?PRIVATEFLAG?
      **
      ** Server announces that it has a particular file.  If this is
      ** not a file that we have and we are pulling, then create a







>
|
>
>







2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
      if( blob_eq(&xfer.aToken[0], "gimme")
       && xfer.nToken==2
       && blob_is_hname(&xfer.aToken[1])
      ){
        remote_unk(&xfer.aToken[1]);
        if( syncFlags & SYNC_PUSH ){
          int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
          if( rid ){
            send_file(&xfer, rid, &xfer.aToken[1], 0);
            nGimmeRcvd++;
          }
        }
      }else

      /*   igot HASH  ?PRIVATEFLAG?
      **
      ** Server announces that it has a particular file.  If this is
      ** not a file that we have and we are pulling, then create a