Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | New command 'state foreachrow' for incremental result processing, using less memory. Converted a number of places in pass InitCSet to this command, and marked a number of othre places for possible future use. |
|---|---|
| Downloads: | Tarball | ZIP archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
6559f3231e6964c3beed30174498639c |
| User & Date: | aku 2008-02-24 04:43:56.000 |
Context
|
2008-02-24
| ||
| 18:01 | Updated my notes regarding memory usage. Converted more locations to incremental query processing via 'state foreachrow', now throughout the importer. ... (check-in: f637d42206 user: aku tags: trunk) | |
| 04:43 | New command 'state foreachrow' for incremental result processing, using less memory. Converted a number of places in pass InitCSet to this command, and marked a number of othre places for possible future use. ... (check-in: 6559f3231e user: aku tags: trunk) | |
| 02:16 | Plugged memory leak in changeset destructor. Updated commentary. Reformatting of a few integrity checks for readability. ... (check-in: 4b0f43fb2f user: aku tags: trunk) | |
Changes
Changes to tools/cvs2fossil/lib/c2f_pinitcsets.tcl.
| ︙ | ︙ | |||
193 194 195 196 197 198 199 | # early, extending them with all their revisions. This # however would mean lots of (slow) method invokations # on the csets. Doing it like this, late creation, means # less such calls. None, but the creation itself. log write 14 initcsets meta_begin mem::mark | | | > > | > | | | | | | 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 |
# early, extending them with all their revisions. This
# however would mean lots of (slow) method invokations
# on the csets. Doing it like this, late creation, means
# less such calls. None, but the creation itself.
log write 14 initcsets meta_begin
mem::mark
state foreachrow {
SELECT M.mid AS xmid,
R.rid AS xrid,
M.pid AS xpid
FROM revision R,
meta M -- R ==> M, using PK index of M.
WHERE R.mid = M.mid
ORDER BY M.mid, R.date
} {
log write 14 initcsets meta_next
if {$lastmeta != $xmid} {
if {[llength $revisions]} {
incr n
set p [repository projectof $lastproject]
log write 14 initcsets meta_cset_begin
mem::mark
set cset [project::rev %AUTO% $p rev $lastmeta $revisions]
log write 14 initcsets meta_cset_done
set spawned [$cset breakinternaldependencies nx]
$cset persist
$cset destroy
foreach cset $spawned { $cset persist ; $cset destroy }
mem::mark
set revisions {}
}
set lastmeta $xmid
set lastproject $xpid
}
lappend revisions $xrid
}
if {[llength $revisions]} {
incr n
set p [repository projectof $lastproject]
log write 14 initcsets meta_cset_begin
mem::mark
|
| ︙ | ︙ | |||
264 265 266 267 268 269 270 |
# First process the tags, then the branches. We know that
# their ids do not overlap with each other.
set lastsymbol {}
set lastproject {}
set tags {}
| | | > > > | | | | | | | | > > > | | | | | | | 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 |
# First process the tags, then the branches. We know that
# their ids do not overlap with each other.
set lastsymbol {}
set lastproject {}
set tags {}
state foreachrow {
SELECT S.sid AS xsid,
T.tid AS xtid,
S.pid AS xpid
FROM tag T,
symbol S -- T ==> R/S, using PK indices of R, S.
WHERE T.sid = S.sid
ORDER BY S.sid, T.tid
} {
if {$lastsymbol != $xsid} {
if {[llength $tags]} {
incr n
set p [repository projectof $lastproject]
set cset [project::rev %AUTO% $p sym::tag $lastsymbol $tags]
set tags {}
$cset persist
$cset destroy
}
set lastsymbol $xsid
set lastproject $xpid
}
lappend tags $xtid
}
if {[llength $tags]} {
incr n
set p [repository projectof $lastproject]
set cset [project::rev %AUTO% $p sym::tag $lastsymbol $tags]
$cset persist
$cset destroy
}
set lastsymbol {}
set lasproject {}
set branches {}
state foreachrow {
SELECT S.sid AS xsid,
B.bid AS xbid,
S.pid AS xpid
FROM branch B,
symbol S -- B ==> R/S, using PK indices of R, S.
WHERE B.sid = S.sid
ORDER BY S.sid, B.bid
} {
if {$lastsymbol != $xsid} {
if {[llength $branches]} {
incr n
set p [repository projectof $lastproject]
set cset [project::rev %AUTO% $p sym::branch $lastsymbol $branches]
set branches {}
$cset persist
$cset destroy
}
set lastsymbol $xsid
set lastproject $xpid
}
lappend branches $xbid
}
if {[llength $branches]} {
incr n
set p [repository projectof $lastproject]
set cset [project::rev %AUTO% $p sym::branch $lastsymbol $branches]
$cset persist
|
| ︙ | ︙ |
Changes to tools/cvs2fossil/lib/c2f_prev.tcl.
| ︙ | ︙ | |||
114 115 116 117 118 119 120 121 122 123 124 125 126 127 |
# predecessors of dropped changesets. For them we have to
# remove their existing information first before inserting the
# new data.
state run {
DELETE FROM cssuccessor WHERE cid = $myid;
}
set loop 0
foreach nid [$mytypeobj cs_successors $myitems] {
state run {
INSERT INTO cssuccessor (cid, nid)
VALUES ($myid,$nid)
}
if {$nid == $myid} { set loop 1 }
}
| > > | 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 |
# predecessors of dropped changesets. For them we have to
# remove their existing information first before inserting the
# new data.
state run {
DELETE FROM cssuccessor WHERE cid = $myid;
}
set loop 0
# TODO: Check other uses of cs_sucessors.
# TODO: Consider merging cs_sucessor's SELECT with the INSERT here.
foreach nid [$mytypeobj cs_successors $myitems] {
state run {
INSERT INTO cssuccessor (cid, nid)
VALUES ($myid,$nid)
}
if {$nid == $myid} { set loop 1 }
}
|
| ︙ | ︙ | |||
321 322 323 324 325 326 327 328 329 330 331 332 333 334 |
}
return
}
proc Getrevisioninfo {revisions} {
set theset ('[join $revisions {','}]')
set revisions {}
foreach {frid path fname revnr rop} [state run [subst -nocommands -nobackslashes {
SELECT U.uuid, F.visible, F.name, R.rev, R.op
FROM revision R, revuuid U, file F
WHERE R.rid IN $theset -- All specified revisions
AND U.rid = R.rid -- get fossil uuid of revision
AND F.fid = R.fid -- get file of revision
}]] {
| > | 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 |
}
return
}
proc Getrevisioninfo {revisions} {
set theset ('[join $revisions {','}]')
set revisions {}
#foreachrow
foreach {frid path fname revnr rop} [state run [subst -nocommands -nobackslashes {
SELECT U.uuid, F.visible, F.name, R.rev, R.op
FROM revision R, revuuid U, file F
WHERE R.rid IN $theset -- All specified revisions
AND U.rid = R.rid -- get fossil uuid of revision
AND F.fid = R.fid -- get file of revision
}]] {
|
| ︙ | ︙ | |||
579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 |
typemethod inorder {projectid} {
# Return all revision changesets for the specified project, in
# the order given to them by the sort passes. Both the
# filtering by project and sorting make use of 'project::rev
# rev' impossible.
set res {}
foreach {cid cdate} [state run {
SELECT C.cid, T.date
FROM changeset C, cstimestamp T
WHERE C.type = 0 -- limit to revision changesets
AND C.pid = $projectid -- limit to changesets in project
AND T.cid = C.cid -- get ordering information
ORDER BY T.date -- sort into commit order
}] {
lappend res $myidmap($cid) $cdate
}
return $res
}
typemethod getcstypes {} {
| > | | | | > > > | | | 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 |
typemethod inorder {projectid} {
# Return all revision changesets for the specified project, in
# the order given to them by the sort passes. Both the
# filtering by project and sorting make use of 'project::rev
# rev' impossible.
set res {}
#foreachrow
foreach {cid cdate} [state run {
SELECT C.cid, T.date
FROM changeset C, cstimestamp T
WHERE C.type = 0 -- limit to revision changesets
AND C.pid = $projectid -- limit to changesets in project
AND T.cid = C.cid -- get ordering information
ORDER BY T.date -- sort into commit order
}] {
lappend res $myidmap($cid) $cdate
}
return $res
}
typemethod getcstypes {} {
state foreachrow {
SELECT tid, name FROM cstype;
} { set mycstype($name) $tid }
return
}
typemethod load {repository} {
set n 0
log write 2 csets {Loading the changesets}
state foreachrow {
SELECT C.cid AS id,
C.pid AS xpid,
CS.name AS cstype,
C.src AS srcid
FROM changeset C, cstype CS
WHERE C.type = CS.tid
ORDER BY C.cid
} {
log progress 2 csets $n {}
set r [$type %AUTO% [$repository projectof $xpid] $cstype $srcid [state run {
SELECT C.iid
FROM csitem C
WHERE C.cid = $id
ORDER BY C.pos
}] $id]
incr n
}
|
| ︙ | ︙ | |||
890 891 892 893 894 895 896 897 898 899 900 901 902 903 |
# Pull the timestamps for all revisions in the changesets and
# compute their deltas for use by the break finder.
array set delta {}
array set stamp {}
set theset ('[join $revisions {','}]')
foreach {rid time} [state run [subst -nocommands -nobackslashes {
SELECT R.rid, R.date
FROM revision R
WHERE R.rid IN $theset
}]] {
set stamp($rid) $time
}
| > | 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 |
# Pull the timestamps for all revisions in the changesets and
# compute their deltas for use by the break finder.
array set delta {}
array set stamp {}
set theset ('[join $revisions {','}]')
#foreachrow
foreach {rid time} [state run [subst -nocommands -nobackslashes {
SELECT R.rid, R.date
FROM revision R
WHERE R.rid IN $theset
}]] {
set stamp($rid) $time
}
|
| ︙ | ︙ | |||
1160 1161 1162 1163 1164 1165 1166 |
# the various cases. This piece is special in that it
# restricts the successors we look for to the same set of
# revisions we start from. Sensible as we are looking for
# changeset internal dependencies.
array set dep {}
| | | | | | | | | | 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 |
# the various cases. This piece is special in that it
# restricts the successors we look for to the same set of
# revisions we start from. Sensible as we are looking for
# changeset internal dependencies.
array set dep {}
state foreachrow [subst -nocommands -nobackslashes {
-- (1) Primary child
SELECT R.rid AS xrid, R.child AS xchild
FROM revision R
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND R.child IS NOT NULL -- Has primary child
AND R.child IN $theset -- Which is also of interest
UNION
-- (2) Secondary (branch) children
SELECT R.rid AS xrid, B.brid AS xchild
FROM revision R, revisionbranchchildren B
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND R.rid = B.rid -- Select subset of branch children
AND B.brid IN $theset -- Which is also of interest
UNION
-- (4) Child of trunk root successor of last NTDB on trunk.
SELECT R.rid AS xrid, RA.child AS xchild
FROM revision R, revision RA
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND R.isdefault -- Restrict to NTDB
AND R.dbchild IS NOT NULL -- and last NTDB belonging to trunk
AND RA.rid = R.dbchild -- Go directly to trunk root
AND RA.child IS NOT NULL -- Has primary child.
AND RA.child IN $theset -- Which is also of interest
}] {
# Consider moving this to the integrity module.
integrity assert {$xrid != $xchild} {Revision $xrid depends on itself.}
lappend dependencies($xrid) $xchild
set dep($xrid,$xchild) .
}
# The sql statements above looks only for direct dependencies
# between revision in the changeset. However due to the
# vagaries of meta data it is possible for two revisions of
# the same file to end up in the same changeset, without a
# direct dependency between them. However we know that there
|
| ︙ | ︙ | |||
1218 1219 1220 1221 1222 1223 1224 |
# handle this.
log write 14 csets {internal [array size dep]}
log write 14 csets {collected [array size dependencies]}
log write 14 csets pseudo-internalsuccessors
array set fids {}
| | | | | 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 |
# handle this.
log write 14 csets {internal [array size dep]}
log write 14 csets {collected [array size dependencies]}
log write 14 csets pseudo-internalsuccessors
array set fids {}
state foreachrow [subst -nocommands -nobackslashes {
SELECT R.rid AS xrid, R.fid AS xfid
FROM revision R
WHERE R.rid IN $theset
}] { lappend fids($xfid) $xrid }
set groups {}
foreach {fid rids} [array get fids] {
if {[llength $rids] < 2} continue
foreach a $rids {
foreach b $rids {
if {$a == $b} continue
|
| ︙ | ︙ | |||
1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 |
# (4) If R is the last of the NTDB revisions which belong to
# the trunk, then the primary child of the trunk root (the
# '1.2' revision) is a successor, if it exists.
# Note that the branches spawned from the revisions, and the
# tags associated with them are successors as well.
foreach {rid child} [state run [subst -nocommands -nobackslashes {
-- (1) Primary child
SELECT R.rid, R.child
FROM revision R
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND R.child IS NOT NULL -- Has primary child
UNION
| > | 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 |
# (4) If R is the last of the NTDB revisions which belong to
# the trunk, then the primary child of the trunk root (the
# '1.2' revision) is a successor, if it exists.
# Note that the branches spawned from the revisions, and the
# tags associated with them are successors as well.
#foreachrow
foreach {rid child} [state run [subst -nocommands -nobackslashes {
-- (1) Primary child
SELECT R.rid, R.child
FROM revision R
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND R.child IS NOT NULL -- Has primary child
UNION
|
| ︙ | ︙ | |||
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 |
AND RA.rid = R.dbchild -- Go directly to trunk root
AND RA.child IS NOT NULL -- Has primary child.
}]] {
# Consider moving this to the integrity module.
integrity assert {$rid != $child} {Revision $rid depends on itself.}
lappend dependencies([list rev $rid]) [list rev $child]
}
foreach {rid child} [state run [subst -nocommands -nobackslashes {
SELECT R.rid, T.tid
FROM revision R, tag T
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND T.rev = R.rid -- Select tags attached to them
}]] {
lappend dependencies([list rev $rid]) [list sym::tag $child]
}
foreach {rid child} [state run [subst -nocommands -nobackslashes {
SELECT R.rid, B.bid
FROM revision R, branch B
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND B.root = R.rid -- Select branches attached to them
}]] {
lappend dependencies([list rev $rid]) [list sym::branch $child]
| > > | 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 |
AND RA.rid = R.dbchild -- Go directly to trunk root
AND RA.child IS NOT NULL -- Has primary child.
}]] {
# Consider moving this to the integrity module.
integrity assert {$rid != $child} {Revision $rid depends on itself.}
lappend dependencies([list rev $rid]) [list rev $child]
}
#foreachrow
foreach {rid child} [state run [subst -nocommands -nobackslashes {
SELECT R.rid, T.tid
FROM revision R, tag T
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND T.rev = R.rid -- Select tags attached to them
}]] {
lappend dependencies([list rev $rid]) [list sym::tag $child]
}
#foreachrow
foreach {rid child} [state run [subst -nocommands -nobackslashes {
SELECT R.rid, B.bid
FROM revision R, branch B
WHERE R.rid IN $theset -- Restrict to revisions of interest
AND B.root = R.rid -- Select branches attached to them
}]] {
lappend dependencies([list rev $rid]) [list sym::branch $child]
|
| ︙ | ︙ | |||
1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 |
typemethod successors {dv branches} {
upvar 1 $dv dependencies
# The first revision committed on a branch, and all branches
# and tags which have it as their prefered parent are the
# successors of a branch.
set theset ('[join $branches {','}]')
foreach {bid child} [state run [subst -nocommands -nobackslashes {
SELECT B.bid, R.rid
FROM branch B, revision R
WHERE B.bid IN $theset -- Restrict to branches of interest
AND B.first = R.rid -- Get first revision on the branch
}]] {
lappend dependencies([list sym::branch $bid]) [list rev $child]
}
foreach {bid child} [state run [subst -nocommands -nobackslashes {
SELECT B.bid, BX.bid
FROM branch B, preferedparent P, branch BX
WHERE B.bid IN $theset -- Restrict to branches of interest
AND B.sid = P.pid -- Get subordinate branches via the
AND BX.sid = P.sid -- prefered parents of their symbols
}]] {
lappend dependencies([list sym::branch $bid]) [list sym::branch $child]
}
foreach {bid child} [state run [subst -nocommands -nobackslashes {
SELECT B.bid, T.tid
FROM branch B, preferedparent P, tag T
WHERE B.bid IN $theset -- Restrict to branches of interest
AND B.sid = P.pid -- Get subordinate tags via the
AND T.sid = P.sid -- prefered parents of their symbols
}]] {
| > > > | 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 |
typemethod successors {dv branches} {
upvar 1 $dv dependencies
# The first revision committed on a branch, and all branches
# and tags which have it as their prefered parent are the
# successors of a branch.
set theset ('[join $branches {','}]')
#foreachrow
foreach {bid child} [state run [subst -nocommands -nobackslashes {
SELECT B.bid, R.rid
FROM branch B, revision R
WHERE B.bid IN $theset -- Restrict to branches of interest
AND B.first = R.rid -- Get first revision on the branch
}]] {
lappend dependencies([list sym::branch $bid]) [list rev $child]
}
#foreachrow
foreach {bid child} [state run [subst -nocommands -nobackslashes {
SELECT B.bid, BX.bid
FROM branch B, preferedparent P, branch BX
WHERE B.bid IN $theset -- Restrict to branches of interest
AND B.sid = P.pid -- Get subordinate branches via the
AND BX.sid = P.sid -- prefered parents of their symbols
}]] {
lappend dependencies([list sym::branch $bid]) [list sym::branch $child]
}
#foreachrow
foreach {bid child} [state run [subst -nocommands -nobackslashes {
SELECT B.bid, T.tid
FROM branch B, preferedparent P, tag T
WHERE B.bid IN $theset -- Restrict to branches of interest
AND B.sid = P.pid -- Get subordinate tags via the
AND T.sid = P.sid -- prefered parents of their symbols
}]] {
|
| ︙ | ︙ |
Changes to tools/cvs2fossil/lib/c2f_state.tcl.
| ︙ | ︙ | |||
156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
return
}
typemethod run {args} {
Save $args
return [uplevel 1 [linsert $args 0 $mystate eval]]
}
typemethod one {args} {
Save $args
return [uplevel 1 [linsert $args 0 $mystate onecolumn]]
}
typemethod transaction {script} {
| > > > > > > | 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 |
return
}
typemethod run {args} {
Save $args
return [uplevel 1 [linsert $args 0 $mystate eval]]
}
typemethod foreachrow {sql script} {
Save $sql
uplevel 1 [list $mystate eval $sql $script]
return
}
typemethod one {args} {
Save $args
return [uplevel 1 [linsert $args 0 $mystate onecolumn]]
}
typemethod transaction {script} {
|
| ︙ | ︙ |