Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | First attempt at transfering SYNCLOG data over the wire. |
|---|---|
| Downloads: | Tarball | ZIP archive |
| Timelines: | family | ancestors | descendants | both | synclog |
| Files: | files | file ages | folders |
| SHA3-256: |
d58eebb77f05255272856765fd57fee5 |
| User & Date: | drh 2021-12-20 09:53:30.738 |
Context
|
2021-12-20
| ||
| 10:04 | Fix a bug in the logic that prevents loopback URLs in the synclog. ... (check-in: 3ef61c4782 user: drh tags: synclog) | |
| 09:53 | First attempt at transfering SYNCLOG data over the wire. ... (check-in: d58eebb77f user: drh tags: synclog) | |
| 08:39 | Merge latest changes from trunk. ... (check-in: 3348c66e3d user: drh tags: synclog) | |
Changes
Changes to src/export.c.
| ︙ | ︙ | |||
1698 1699 1700 1701 1702 1703 1704 |
const char *zRepack = "git repack -adf";
gitmirror_message(VERB_NORMAL, "%s\n", zRepack);
fossil_system(zRepack);
}
/* Record this export into the sync log */
zMirrorAbs = file_canonical_name_dup(zMirror);
| | | 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 |
const char *zRepack = "git repack -adf";
gitmirror_message(VERB_NORMAL, "%s\n", zRepack);
fossil_system(zRepack);
}
/* Record this export into the sync log */
zMirrorAbs = file_canonical_name_dup(zMirror);
sync_log_entry("this", zMirrorAbs, 0, "git");
fossil_free(zMirrorAbs);
/* Optionally do a "git push" */
zPushUrl = db_text(0, "SELECT value FROM mconfig WHERE key='autopush'");
if( zPushUrl ){
char *zPushCmd;
UrlData url;
|
| ︙ | ︙ | |||
1723 1724 1725 1726 1727 1728 1729 |
if( rc ){
fossil_fatal("cannot push content using: %s", zPushCmd);
}else if( db_is_writeable("repository") ){
db_unprotect(PROTECT_CONFIG);
db_multi_exec("REPLACE INTO config(name,value,mtime)"
"VALUES('gitpush:%q',1,now())", zPushUrl);
db_protect_pop();
| | | 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 |
if( rc ){
fossil_fatal("cannot push content using: %s", zPushCmd);
}else if( db_is_writeable("repository") ){
db_unprotect(PROTECT_CONFIG);
db_multi_exec("REPLACE INTO config(name,value,mtime)"
"VALUES('gitpush:%q',1,now())", zPushUrl);
db_protect_pop();
sync_log_entry("this", zPushUrl, 0, "git-push");
}
fossil_free(zPushCmd);
}
}
/*
** Implementation of the "fossil git status" command.
|
| ︙ | ︙ |
Changes to src/sync.c.
| ︙ | ︙ | |||
91 92 93 94 95 96 97 |
azOther[i] = 0;
}
fossil_free(azOther);
return nErr;
}
/*
| | < < < < < < | | > | < < | | | | | | < < < < < < < < < < < < | 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 |
azOther[i] = 0;
}
fossil_free(azOther);
return nErr;
}
/*
** Make a new entry, or update an existing entry, in the SYNCLOG table.
**
** For an ordinary push/pull, zType is NULL. But it may also be a string
** describing non-standard operations. For example zType might be "git"
** when doing a "fossil git export", or zType might be "import" when doing
** a "fossil pull --from-parent-project".
*/
void sync_log_entry(
const char *zFrom, /* Content comes from this machine */
const char *zTo, /* Content goes to this machine */
i64 iTime, /* Transfer time, or 0 for "now" */
const char *zType /* Type of sync. NULL for normal */
){
schema_synclog();
if( iTime<=0 ){
db_multi_exec(
"INSERT INTO repository.synclog(sfrom,sto,stime,stype)"
" VALUES(%Q,%Q,unixepoch(),%Q)"
" ON CONFLICT DO UPDATE SET stime=unixepoch()",
zFrom, zTo, zType
);
}else{
db_multi_exec(
"INSERT INTO repository.synclog(sfrom,sto,stime,stype)"
" VALUES(%Q,%Q,%lld,%Q)"
" ON CONFLICT DO UPDATE SET stime=%lld WHERE stime<%lld",
zFrom, zTo, iTime, zType, iTime, iTime
);
}
}
/*
** If the repository is configured for autosyncing, then do an
** autosync. Bits of the "flags" parameter determine details of behavior:
**
|
| ︙ | ︙ | |||
757 758 759 760 761 762 763 |
}else{
fossil_fatal("backup \"%s\" already exists", zDest);
}
}
db_unprotect(PROTECT_ALL);
db_multi_exec("VACUUM repository INTO %Q", zDest);
zFullName = file_canonical_name_dup(zDest);
| | | 738 739 740 741 742 743 744 745 746 747 |
}else{
fossil_fatal("backup \"%s\" already exists", zDest);
}
}
db_unprotect(PROTECT_ALL);
db_multi_exec("VACUUM repository INTO %Q", zDest);
zFullName = file_canonical_name_dup(zDest);
sync_log_entry("this", zFullName, 0, "backup");
fossil_free(zFullName);
}
|
Changes to src/xfer.c.
| ︙ | ︙ | |||
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 |
** # ... code here
** set common_done 1
** }
*/
int xfer_run_common_script(void){
return xfer_run_script(xfer_common_code(), 0, 0);
}
/*
** If this variable is set, disable login checks. Used for debugging
** only.
*/
static int disableLogin = 0;
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 |
** # ... code here
** set common_done 1
** }
*/
int xfer_run_common_script(void){
return xfer_run_script(xfer_common_code(), 0, 0);
}
/*
** The current line is a "pragma synclog". Accept this line into
** the SYNCLOG table, if appropriate.
**
** pragma synclog FROM TO MTIME ?TYPE?
**
** The name of the remote size is zRemote. The value of "this" for
** either the FROM or TO fields is converted into zRemote.
**
** If either FROM or TO is an alias for the current repository, then
** silently reject the entry.
*/
static void xfer_accept_synclog_pragma(
Xfer *pXfer, /* Current line of xfer script */
const char *zRemote /* The name of the remote repository */
){
const char *zFrom = blob_str(&pXfer->aToken[2]);
const char *zTo = blob_str(&pXfer->aToken[3]);
i64 iTime = strtoll(blob_str(&pXfer->aToken[4]), 0, 0);
const char *zType = pXfer->nToken>=5 ? blob_str(&pXfer->aToken[5]) : 0;
if( strcmp(zFrom, "this")==0 ){
zFrom = zRemote;
}else if( db_exists("SELECT 1 FROM config WHERE name='baseurl:%q'",zFrom) ){
iTime = 0;
}
if( strcmp(zTo, "this")==0 ){
zTo = zRemote;
}else if( db_exists("SELECT 1 FROM config WHERE name='baseurl:%q'",zTo) ){
iTime = 0;
}
if( iTime>0 ) sync_log_entry(zFrom, zTo, iTime, zType);
}
/*
** If this variable is set, disable login checks. Used for debugging
** only.
*/
static int disableLogin = 0;
|
| ︙ | ︙ | |||
1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 |
int rc;
const char *zScript = 0;
char *zUuidList = 0;
int nUuidList = 0;
char **pzUuidList = 0;
int *pnUuidList = 0;
int uvCatalogSent = 0;
if( fossil_strcmp(PD("REQUEST_METHOD","POST"),"POST") ){
fossil_redirect_home();
}
g.zLogin = "anonymous";
login_set_anon_nobody_capabilities();
login_check_credentials();
| > | 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 |
int rc;
const char *zScript = 0;
char *zUuidList = 0;
int nUuidList = 0;
char **pzUuidList = 0;
int *pnUuidList = 0;
int uvCatalogSent = 0;
const char *zClientUrl = 0;
if( fossil_strcmp(PD("REQUEST_METHOD","POST"),"POST") ){
fossil_redirect_home();
}
g.zLogin = "anonymous";
login_set_anon_nobody_capabilities();
login_check_credentials();
|
| ︙ | ︙ | |||
1684 1685 1686 1687 1688 1689 1690 |
blob_str(&xfer.aToken[3])
);
db_protect_pop();
}
if( db_get_boolean("forbid-delta-manifests",0) ){
@ pragma avoid-delta-manifests
}
| | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 |
blob_str(&xfer.aToken[3])
);
db_protect_pop();
}
if( db_get_boolean("forbid-delta-manifests",0) ){
@ pragma avoid-delta-manifests
}
}else
/* pragma ci-unlock CLIENT-ID
**
** Remove any locks previously held by CLIENT-ID. Clients send this
** pragma with their own ID whenever they know that they no longer
** have any commits pending.
*/
if( blob_eq(&xfer.aToken[1], "ci-unlock")
&& xfer.nToken==3
&& blob_is_hname(&xfer.aToken[2])
){
db_unprotect(PROTECT_CONFIG);
db_multi_exec(
"DELETE FROM config"
" WHERE name GLOB 'ci-lock-*'"
" AND json_extract(value,'$.clientid')=%Q",
blob_str(&xfer.aToken[2])
);
db_protect_pop();
}else
/* pragma req-synclog ?CLIENT-URL?
**
** Request synclog data. If the CLIENT-URL argument is provided,
** it will be the canonical URL for the client.
*/
if( blob_eq(&xfer.aToken[1], "req-synclog") && g.perm.RdSLog ){
Stmt qSynclog;
if( xfer.nToken>=2 ){
zClientUrl = blob_str(&xfer.aToken[2]);
if( sqlite3_strlike("http%//localhost%", zClientUrl, 0)==0 ){
zClientUrl = 0;
}
}
db_prepare(&qSynclog,
"SELECT sfrom, sto, unixepoch(stime), stype FROM synclog"
);
while( db_step(&qSynclog)==SQLITE_ROW ){
const char *zFrom = db_column_text(&qSynclog,0);
const char *zTo = db_column_text(&qSynclog,1);
const char *zTime = db_column_text(&qSynclog,2);
const char *zType = db_column_text(&qSynclog,3);
if( zClientUrl ){
if( strcmp(zFrom, zClientUrl)==0 ) continue;
if( strcmp(zTo, zClientUrl)==0 ) continue;
}
if( zType!=0 && zType[0]!=0 ){
@ pragma synclog %s(zFrom) %s(zTo) %s(zTime) %s(zType)
}else{
@ pragma synclog %s(zFrom) %s(zTo) %s(zTime)
}
}
}else
/* pragma synclog FROM TO MTIME ?TYPE?
**
** The client is uploading an entry from its SYNCLOG table. This
** will only be accepted if both:
**
** (1) The client as WrSLog permission
** (2) A prior req-synclog pragma has identified the URL of the client
**
** Insert the entry into the SYNC log if appropriate.
*/
if( xfer.nToken>=4
&& blob_eq(&xfer.aToken[1], "synclog")
&& g.perm.WrSLog
&& zClientUrl!=0
){
xfer_accept_synclog_pragma(&xfer, zClientUrl);
}
}else
/* Unknown message
*/
{
|
| ︙ | ︙ | |||
2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 |
db_lset("client-id", zClientId);
}
blob_appendf(&send, "pragma ci-lock %s %s\n", zCkinLock, zClientId);
zCkinLock = 0;
}else if( zClientId ){
blob_appendf(&send, "pragma ci-unlock %s\n", zClientId);
}
/* Append randomness to the end of the uplink message. This makes all
** messages unique so that that the login-card nonce will always
** be unique.
*/
zRandomness = db_text(0, "SELECT hex(randomblob(20))");
blob_appendf(&send, "# %s\n", zRandomness);
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 |
db_lset("client-id", zClientId);
}
blob_appendf(&send, "pragma ci-lock %s %s\n", zCkinLock, zClientId);
zCkinLock = 0;
}else if( zClientId ){
blob_appendf(&send, "pragma ci-unlock %s\n", zClientId);
}
/* Transfer SYNCLOG data on the first roundtrip, if appropriate */
if( nCycle==0 ){
const char *zSelfUrl = public_url();
if( sqlite3_strlike("http%//localhost%", zSelfUrl, 0)==0 ){
zSelfUrl = 0;
}
if( zSelfUrl==0 ){
blob_appendf(&send,"pragma req-synclog\n");
}else{
blob_appendf(&send,"pragma req-synclog %s\n", zSelfUrl);
if( syncFlags & SYNC_PUSH_SYNCLOG ){
Stmt qSynclog;
db_prepare(&qSynclog,
"SELECT sfrom, sto, unixepoch(stime), stype FROM synclog"
" WHERE sfrom!=%Q AND sto!=%Q",
g.url.canonical, g.url.canonical
);
while( db_step(&qSynclog)==SQLITE_ROW ){
const char *zFrom = db_column_text(&qSynclog,0);
const char *zTo = db_column_text(&qSynclog,1);
const char *zTime = db_column_text(&qSynclog,2);
const char *zType = db_column_text(&qSynclog,3);
if( zType!=0 && zType[0]!=0 ){
blob_appendf(&send,"pragma synclog %s %s %s %s\n",
zFrom, zTo, zTime, zType);
}else{
blob_appendf(&send,"pragma synclog %s %s %s\n",
zFrom, zTo, zTime);
}
}
}
}
}
/* Append randomness to the end of the uplink message. This makes all
** messages unique so that that the login-card nonce will always
** be unique.
*/
zRandomness = db_text(0, "SELECT hex(randomblob(20))");
blob_appendf(&send, "# %s\n", zRandomness);
|
| ︙ | ︙ | |||
2522 2523 2524 2525 2526 2527 2528 |
xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
if( xfer.nToken>=5 ){
xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
}
}
| | | | | 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 |
xfer.remoteVersion = atoi(blob_str(&xfer.aToken[2]));
if( xfer.nToken>=5 ){
xfer.remoteDate = atoi(blob_str(&xfer.aToken[3]));
xfer.remoteTime = atoi(blob_str(&xfer.aToken[4]));
}
}
/* pragma synclog FROM TO MTIME ?TYPE?
**
** The server is downloading an entry from its SYNCLOG table. Merge
** this into the local SYNCLOG table if appropriate.
** is running. The DATE and TIME are a pure numeric ISO8601 time
** for the specific check-in of the client.
*/
if( xfer.nToken>=4 && blob_eq(&xfer.aToken[1], "synclog") ){
xfer_accept_synclog_pragma(&xfer, g.url.canonical);
}
/* pragma uv-pull-only
** pragma uv-push-ok
**
** If the server is unwill to accept new unversioned content (because
** this client lacks the necessary permissions) then it sends a
|
| ︙ | ︙ | |||
2703 2704 2705 2706 2707 2708 2709 |
}else if( rSkew*24.0*3600.0 < -10.0 ){
fossil_warning("*** time skew *** server is slow by %s",
db_timespan_name(-rSkew));
g.clockSkewSeen = 1;
}
if( nErr==0 ){
| > > > > | > > > > > | 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 |
}else if( rSkew*24.0*3600.0 < -10.0 ){
fossil_warning("*** time skew *** server is slow by %s",
db_timespan_name(-rSkew));
g.clockSkewSeen = 1;
}
if( nErr==0 ){
if( zAltPCode!=0 ){
sync_log_entry(g.url.canonical, "this", 0, "import");
}else{
if( syncFlags & SYNC_PUSH ){
sync_log_entry("this", g.url.canonical, 0, 0);
}
if( syncFlags & SYNC_PULL ){
sync_log_entry(g.url.canonical, "this", 0, 0);
}
}
}
fossil_force_newline();
fossil_print(
"%s done, wire bytes sent: %lld received: %lld ip: %s\n",
zOpType, nSent, nRcvd, g.zIpAddr);
if( syncFlags & SYNC_VERBOSE ){
|
| ︙ | ︙ |