AppFS
Check-in [bfaf95df49]
Login
HomeTimelineFilesBranchesTagsTicketsWiki
Overview
Comment:More work towards actually doing multiple hashing algorithms
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | complete-multihash
Files: files | file ages | folders
SHA1: bfaf95df49a158a4635b394c126717ad880d8f60
User & Date: rkeene on 2017-01-20 17:19:04
Other Links: branch diff | manifest | tags
Context
2017-01-20
17:19
More work towards actually doing multiple hashing algorithms Leaf check-in: bfaf95df49 user: rkeene tags: complete-multihash
2017-01-19
17:49
Started work on completely supporting multiple hashing algorithms check-in: 2460a1ddab user: rkeene tags: complete-multihash
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Modified README.md from [35b476a0b0] to [a25b7811db].

︙
14
15
16
17
18
19
20
21
22


23
24
25
26


27
28
29
30
31

32
33
34


35
36
37

38
39


40
41
42
43

14
15
16
17
18
19
20


21
22
23
24


25
26
27
28
29
30

31
32


33
34
35
36
37
38


39
40
41
42
43
44








-
-
+
+


-
-
+
+




-
+

-
-
+
+



+
-
-
+
+





    AppFS should normally be mounted on "/opt/appfs".

    /opt/appfs/hostname
    	Fetches: http://hostname/appfs/index
    	Contains CSV file: hash,hashMethod,<certificateInDERFormatInHex>,<PKCS#1v1.5-signature-inDERFormatInHex>
	                   \-------------/
                                  ^- Signed data
    	Fetches: http://hostname/appfs/sha1/<hash>
    	Contains CSV file: package,version,os,cpuArch,sha1,isLatest
    	Fetches: http://hostname/appfs/<hashMethod>/<hash>
    	Contains CSV file: package,version,os,cpuArch,packageManifestHash,isLatest

    /opt/appfs/hostname/package/os-cpuArch/version
    /opt/appfs/hostname/sha1/
    	Fetches: http://hostname/appfs/sha1/<sha1>
    /opt/appfs/hostname/<hashMethod>/
    	Fetches: http://hostname/appfs/<hashMethod>/<packageManifestHash>
    	Contains CSV file:
    		type,time,extraData,name
    		type == directory; extraData = (null)
    		type == symlink; extraData = source
    		type == file; extraData = size,perms,sha1
    		type == file; extraData = size,perms,fileHash

    /opt/appfs/hostname/{sha1,package/os-cpuArch/version}/file
    	Fetches: http://hostname/appfs/sha1/<sha1>
    /opt/appfs/hostname/{packageManifestHash,package/os-cpuArch/version}/file
    	Fetches: http://hostname/appfs/<hashMethod>/<fileHash>

Database
--------
    sites(hostname, hashMethod, lastUpdate, ttl)
    packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest)
    files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory)
    packages(hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest)
    files(packageManifestHash, type, time, source, size, perms, fileHash, file_name, file_directory)

Resources
---------
http://appfs.rkeene.org/

Modified appfsd.tcl from [bd58d61afc] to [fbd0fd32b5].

︙
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184

185
186
187
188
189
190
191
192
193

194
195
196
197
198
199
200

166
167
168
169
170
171
172




173
174
175
176
177
178
179

180
181
182
183
184
185
186
187
188

189
190
191
192
193
194
195
196








-
-
-
-







-
+








-
+








		return $file
	}


	proc _isHash {value} {
		set value [string tolower $value]

		if {[string length $value] != 40} {
			return false
		}

		if {![regexp {^[0-9a-f]*$} $value]} {
			return false
		}

		return true
	}

	proc _verifySignatureAndCertificate {hostname certificate signature hash} {
	proc _verifySignatureAndCertificate {hostname certificate signature hash hashmethod} {
		set certificate [binary format "H*" $certificate]
		set signature   [binary format "H*" $signature]

		set certificate [::pki::x509::parse_cert $certificate]

		array set certificate_arr $certificate
		set certificate_cn [::pki::x509::_dn_to_cn $certificate_arr(subject)]

		if {![::pki::verify $signature "$hash,sha1" $certificate]} {
		if {![::pki::verify $signature "$hash,$hashmethod" $certificate]} {
			return false
		}

		if {[string tolower $certificate_cn] != [string tolower $hostname]} {
			return false
		}
︙
310
311
312
313
314
315
316
317
318
319



320
321
322
323
324


325
326
327
328
329
330
331

306
307
308
309
310
311
312



313
314
315
316
317
318


319
320
321
322
323
324
325
326
327








-
-
-
+
+
+



-
-
+
+









			sqlite3 ::appfs::db [file join $::appfs::cachedir cache.db]

			::appfs::db timeout 30000
		}

		# Create tables
		db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, lastUpdate, ttl);}
		db eval {CREATE TABLE IF NOT EXISTS packages(hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest);}
		db eval {CREATE TABLE IF NOT EXISTS files(package_sha1, type, time, source, size, perms, file_sha1, file_name, file_directory);}
		db eval {CREATE TABLE IF NOT EXISTS sites(hostname PRIMARY KEY, hashMethod, lastUpdate, ttl);}
		db eval {CREATE TABLE IF NOT EXISTS packages(hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest);}
		db eval {CREATE TABLE IF NOT EXISTS files(packageManifestHash, hashMethod, type, time, source, size, perms, fileHash, file_name, file_directory);}

		# Create indexes
		db eval {CREATE INDEX IF NOT EXISTS sites_index ON sites (hostname);}
		db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, sha1, package, version, os, cpuArch);}
		db eval {CREATE INDEX IF NOT EXISTS files_index ON files (package_sha1, file_name, file_directory);}
		db eval {CREATE INDEX IF NOT EXISTS packages_index ON packages (hostname, packageManifestHash, package, version, os, cpuArch);}
		db eval {CREATE INDEX IF NOT EXISTS files_index ON files (packageManifestHash, file_name, file_directory);}
	}

	proc download {hostname hash {method sha1}} {
		set url [::appfs::user::construct_url $hostname $hash $method]
		set file [_cachefile $url $hash $method]

		if {![file exists $file]} {
︙
376
377
378
379
380
381
382
383

384
385
386
387

388
389
390
391
392
393
394

372
373
374
375
376
377
378

379
380
381
382

383
384
385
386
387
388
389
390








-
+



-
+








		set indexhashcert   [lindex $indexhash_data 2]
		set indexhashsig    [lindex $indexhash_data 3]

		if {![_isHash $indexhash]} {
			return -code error "Invalid hash: $indexhash"
		}

		if {![_verifySignatureAndCertificate $hostname $indexhashcert $indexhashsig $indexhash]} {
		if {![_verifySignatureAndCertificate $hostname $indexhashcert $indexhashsig $indexhash $indexhashmethod]} {
			return -code error "Invalid signature or certificate from $hostname"
		}

		set file [download $hostname $indexhash]
		set file [download $hostname $indexhash $indexhashmethod]
		catch {
			set fd [open $file]
		}

		if {![info exists fd]} {
			return -code error "Unable to download or open $file"
		}
︙
417
418
419
420
421
422
423
424

425
426
427
428
429
430
431
432
433
434
435
436
437

438
439
440
441
442
443
444
445
446

447
448
449
450
451
452
453
454
455
456
457
458
459
460

461
462
463

464
465
466
467
468
469
470
471



472
473
474
475
476
477

478
479
480
481

482
483
484
485
486
487
488

413
414
415
416
417
418
419

420
421
422
423
424
425
426
427
428
429
430
431
432

433
434
435
436
437
438
439
440
441

442
443
444
445
446
447
448
449
450
451
452
453
454
455

456
457
458

459
460
461
462
463
464
465


466
467
468
469
470
471
472
473

474
475
476
477

478
479
480
481
482
483
484
485








-
+












-
+








-
+













-
+


-
+






-
-
+
+
+





-
+



-
+








			unset -nocomplain pkgInfo
			if {[catch {
				set pkgInfo(package)  [lindex $work 0]
				set pkgInfo(version)  [lindex $work 1]
				set pkgInfo(os)       [_normalizeOS [lindex $work 2]]
				set pkgInfo(cpuArch)  [_normalizeCPU [lindex $work 3]]
				set pkgInfo(hash)     [string tolower [lindex $work 4]]
				set pkgInfo(hash_type) "sha1"
				set pkgInfo(hash_type) $indexhashmethod
				set pkgInfo(isLatest) [expr {!![lindex $work 5]}]
			}]} {
				continue
			}

			if {![_isHash $pkgInfo(hash)]} {
				continue
			}

			lappend curr_packages $pkgInfo(hash)

			# Do not do any additional work if we already have this package
			set existing_packages [db eval {SELECT package FROM packages WHERE hostname = $hostname AND sha1 = $pkgInfo(hash);}]
			set existing_packages [db eval {SELECT package FROM packages WHERE hostname = $hostname AND packageManifestHash = $pkgInfo(hash);}]
			if {[lsearch -exact $existing_packages $pkgInfo(package)] != -1} {
				continue
			}

			if {$pkgInfo(isLatest)} {
				db eval {UPDATE packages SET isLatest = 0 WHERE hostname = $hostname AND package = $pkgInfo(package) AND os = $pkgInfo(os) AND cpuArch = $pkgInfo(cpuArch);}
			}

			db eval {INSERT INTO packages (hostname, sha1, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
			db eval {INSERT INTO packages (hostname, packageManifestHash, package, version, os, cpuArch, isLatest, haveManifest) VALUES ($hostname, $pkgInfo(hash), $pkgInfo(package), $pkgInfo(version), $pkgInfo(os), $pkgInfo(cpuArch), $pkgInfo(isLatest), 0);}
		}

		# Look for packages that have been deleted
		set found_packages [db eval {SELECT sha1 FROM packages WHERE hostname = $hostname;}]
		foreach package $found_packages {
			set found_packages_arr($package) 1
		}

		foreach package $curr_packages {
			unset -nocomplain found_packages_arr($package)
		}

		foreach package [array names found_packages_arr] {
			db eval {DELETE FROM packages WHERE hostname = $hostname AND sha1 = $package;}
			db eval {DELETE FROM packages WHERE hostname = $hostname AND packageManifestHash = $package;}
		}

		db eval {INSERT OR REPLACE INTO sites (hostname, lastUpdate, ttl) VALUES ($hostname, $now, $::appfs::ttl);}
		db eval {INSERT OR REPLACE INTO sites (hostname, hashMethod, lastUpdate, ttl) VALUES ($hostname, $indexhashmethod, $now, $::appfs::ttl);}

		appfsd::get_path_info_cache_flush

		return COMPLETE
	}

	proc getpkgmanifest {hostname package_sha1} {
		set haveManifest [db onecolumn {SELECT haveManifest FROM packages WHERE sha1 = $package_sha1 LIMIT 1;}]
	proc getpkgmanifest {hostname packageManifestHash} {
		set haveManifest [db onecolumn {SELECT haveManifest FROM packages WHERE packageManifestHash = $packageManifestHash LIMIT 1;}]
		set siteHashMethod [db onecolumn {SELECT hashMethod FROM sites WHERE hostname = $hostname LIMIT 1;}

		if {$haveManifest == "1"} {
			return COMPLETE
		}

		if {![_isHash $package_sha1]} {
		if {![_isHash $packageManifestHash]} {
			return FAIL
		}

		set file [download $hostname $package_sha1]
		set file [download $hostname $packageManifestHash $siteHashMethod]

		catch {
			set fd [open $file]
		}

		if {![info exists fd]} {
			return -code error "Unable to download or open $file"
︙

Powered by Fossil · RSS