libinterp/keyring.h
libinterp/loadermod.h
libinterp/mathmod.h
libinterp/runt.h
libinterp/sysmod.h
libinterp/tkmod.h
limbo/runt.h

services/httpd/httpd.debug
services/httpd/httpd.log
tomo-*-*-*-*.zip

pipeline:
  Linux-386--debian:
    image: debian:stable-slim
    group: build
    commands:
      - dpkg --add-architecture i386
      - apt update >/dev/null
      - apt install -yy curl:i386 binutils:i386 gcc:i386 libc6-dev-i386 libx11-dev:i386 libxext-dev:i386 zip:i386 >/dev/null
      - ./utils/tomo-dev.sh build-Linux `pwd`
      - rm -rf .git .hg
      - zip -r "tomo-Linux-386--debian-$CI_COMMIT_SHA-$CI_BUILD_STARTED.zip" *
      - ./utils/tomo-dev.sh upload-ci "tomo-Linux-386--debian-$CI_COMMIT_SHA-$CI_BUILD_STARTED.zip"
#  Linux-386--ubuntu:
#    image: ubuntu:bionic
#    group: build

/* vi: ts=8
 * FreeBSD 386 fpu support
 * Mimic Plan9 floating point support
 */



















































/* FCR */
#define	FPINEX	(1<<5)
#define	FPUNFL	((1<<4)|(1<<1))
#define	FPOVFL	(1<<3)
#define	FPZDIV	(1<<2)
#define	FPINVAL	(1<<0)
#define	FPRNR	(0<<10)

waitpid0, waitpid1 : int;
mainpid : int;

fontcache : array of ref Reffont;
nfontcache : int;
reffonts : array of ref Reffont;
deffontnames := array[2] of { # TODO theme support
	"/fonts/hermit/regular/16/font",
	"/fonts/BQN386/16/font",
};

command : ref Command;

WPERCOL : con 8;

NSnarf : con 32;

install:V:	$ROOT/dis/charon.dis

$ROOT/dis/charon.dis:	charon.dis
	rm -f $target && cp charon.dis $target

charon.dis:	$MODULES $SYS_MODULES

# see </tktview/c235634a814ed923f35e6aae88f18b36e35716b3>
#img.dis:	img.b $MODULE $SYS_MODULE
#	limbo $LIMBOFLAGS -c -gw img.b

nuke:V:
	rm -f $ROOT/dis/charon.dis

<../../../mkconfig

DIRS=\
	ppp\
#	nppp\

TARG=\
	bootpd.dis\
	dhcp.dis\
	ping.dis\
	rip.dis\
	shttpd.dis\
	tftpd.dis\
	virgild.dis\
	sntp.dis\

SYSMODULES=\
	arg.m\
	attrdb.m\
	bufio.m\
	daytime.m\
	dhcp.m\
	draw.m\
	encoding.m\
	env.m\
	ether.m\
	ip.m\
	ipattr.m\
	keyring.m\
	regex.m\
	mhttp.m\
	security.m\
	sh.m\
	string.m\
	sys.m\

DISBIN=$ROOT/dis/ip

<$ROOT/mkfiles/mkdis
<$ROOT/mkfiles/mksubdirs

# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
# <6972b3440c27a83c1a56c5ecc781148514fb1f11>
# <https://github.com/mjl-/httpd/>
# <https://git.heropunch.io/ar/inferno-httpd/>
implement Shttpd;

# features:
#
# - http/1.0, http/1.1 (keep-alive, chunked responses).  no "http/0.9".
# - cgi and scgi support
# - virtual hosts
# - determining content-type based on file extension
# - error/access logging
# - index file serving, html directory listings
# - http basic authentication
# - "range" requests, for random access to files
# - redirections with regular expressions
# - configuration in attrdb format
# - as compliant as possible, where reasonable
# - mindful of security
#
# see the manual page for full documentation
#
# non-features:
#
# - connection rate limiting.  should be done at a higher level, e.g. firewall.
# - silly "correct" behavior per RFC
#
# testing/todo:
#
# - doc/shttpd-testing.txt can be used for testing.  it has to be done manually,
#   enough corner cases have not been tested, but quite a few have.
# - when responding with chunked data, can we embed an error message in a chunks
#   comment-part when an error with reading the data occurs?
# - support transfer-encoding chunked and compressed content-encodings from client?
# - check for security issues with http
# - think of ssl?
# - try to fix dir listings for paths with multiple slashes in them.  firefox breaks
#   on them, is firefox right?
# - test with other browsers

include "sys.m";
	sys: Sys;
	print, sprint, fprint, fildes: import sys;
include "draw.m";
include "arg.m";
include "bufio.m";
	bufio: Bufio;
	Iobuf: import bufio;
include "daytime.m";
	daytime: Daytime;
include "env.m";
	env: Env;
include "string.m";
	str: String;
include "keyring.m";
	keyring: Keyring;
include "security.m";
	random: Random;
include "encoding.m";
	base64: Encoding;
include "sh.m";
	sh: Sh;
include "ip.m";
	ipm: IP;
	IPaddr: import ipm;
include "attrdb.m";
	attrdb: Attrdb;
	Db, Dbentry, Tuples: import attrdb;
include "regex.m";
	regex: Regex;
include "mhttp.m";
	http: Http;
	Url, Req, Resp, Hdrs, HTTP_10, HTTP_11, encodepath: import http;
	UNKNOWN, OPTIONS, GET, HEAD, POST, PUT, DELETE, TRACE, CONNECT: import http;

Version: con "~tomo/shttpd/0";


Repl: adt {
	re:	regex->Re;
	rule:	list of ref (string, int);	# literal string, replacement group for regex
	maxrepl:	int;	# highest replacement group

	parse:	fn(restr, rulestr: string): (ref Repl, string);
	apply:	fn(r: self ref Repl, s: string): (int, string, string);
};

# config for a single host,port
Cfg: adt {
	host, port:	string;
	listings:	array of ref (string, int);  # path prefix, list (true/false)
	cachesecs:	int;
	listens:	list of ref (string, string);	# ip, port
	cgipaths:	list of ref (string, string, list of string, int);	# path, cmd|addr, methods, Cgi|Scgi
	indexfiles:	list of string;
	redirs:	list of ref Repl;
	auths:	list of ref (string, string, string);	# path, realm, base64 user:pass
	gzpaths:	list of string;

	new:	fn(): ref Cfg;
	read:	fn(e: ref Dbentry, defaultport: string): (ref Cfg, string);
	rev:	fn(cfg: self ref Cfg);
};

# config file, holds one Cfg per host,port
Cfgs: adt {
	file:	string;
	db:	ref Db;
	default:	ref Cfg;
	cfgs:	list of (string, string, ref Cfg);	# host, port (for lookup) => config

	accessfd:	ref Sys->FD;
	debugflag, vhostflag:	int;
	addrs:	list of string;
	usertypes:	list of ref (string, string);

	init:	fn(file: string): (ref Cfgs, string);
	read:	fn(file: string, db: ref Db): (ref Cfgs, string);
	find:	fn(c: self ref Cfgs, host, port: string): ref Cfg;
};

# represents a connection and a request on it
Op: adt {
	id:	int;	# connection id, for logging
	now:	int;	# time of start
	keepalive:	int;
	chunked:	int;
	length:		big;
	fd:	ref Sys->FD;
	inb:	ref Bufio->Iobuf;
	rhost, rport, lhost, lport:	string;
	req:	ref Req;
	resp:	ref Resp;
	cfgs:	ref Cfgs;
	cfg:	ref Cfg;
};

Cgitimeoutsecs: con 3*60;
Keepalivesecs: con 3*60;
defaddr: con "net!*!http";

debugflag: int;

webroot: string;
credempty: string;
ctlchan := "";

environment: list of (string, string);

Shttpd: module {
	init:	fn(nil: ref Draw->Context, args: list of string);
};

mimetypes := array[] of {
	(".avi",	"video/x-msvideo"),
	(".b",		"text/plain; charset=utf-8"),
	(".bmp",	"image/bmp"),
	(".bz2",	"application/x-bzip"),
	(".c",		"text/plain; charset=utf-8"),
	(".conf",	"text/plain; charset=utf-8"),
	(".css",	"text/css"),
	(".diff",	"text/plain; charset=utf-8"),
	(".doc",	"application/msword"),
	(".dvi",	"application/x-dvi"),
	(".eps",	"application/postscript"),
	(".gif",	"image/gif"),
	(".gz",		"application/x-gzip"),
	(".h",		"text/plain; charset=utf-8"),
	(".htm",	"text/html; charset=utf-8"),
	(".html",	"text/html; charset=utf-8"),
	(".ico",	"image/x-icon"),
	(".jpeg",	"image/jpeg"),
	(".jpg",	"image/jpeg"),
	(".js",		"text/javascript; charset=utf-8"),
	(".man",	"text/plain; charset=utf-8"),
	(".me",		"text/plain; charset=utf-8"),
	(".mp3",	"audio/mpeg"),
	(".mp4",	"video/mp4"),
	(".mpeg",	"video/mpeg"),
	(".mpg",	"video/mpeg"),
	(".ms",		"text/plain; charset=utf-8"),
	(".ogg",	"application/ogg"),
	(".orig",	"text/plain; charset=utf-8"),
	(".patch",	"text/plain; charset=utf-8"),
	(".pdf",	"application/pdf"),
	(".png",	"image/png"),
	(".ppt",	"application/vnd.ms-powerpoint"),
	(".ps",		"application/postscript"),
	(".py",		"text/plain; charset=utf-8"),
	(".roff",	"text/plain; charset=utf-8"),
	(".sh",		"text/plain; charset=utf-8"),
	(".svg",	"image/svg+xml"),
	(".t",		"text/plain; charset=utf-8"),
	(".tar",	"application/x-tar"),
	(".tar.bz2",	"application/x-bzip-compressed-tar"),
	(".tar.gz",	"application/x-tgz"),
	(".tbz",	"application/x-bzip-compressed-tar"),
	(".tex",	"text/plain; charset=utf-8"),
	(".tgz",	"application/x-tgz"),
	(".tif",	"image/tiff"),
	(".tiff",	"image/tiff"),
	(".torrent",	"application/x-bittorrent"),
	(".txt",	"text/plain; charset=utf-8"),
	(".wav",	"audio/x-wav"),
	(".woff",	"application/font-woff"),
	(".woff2",	"font/woff2"),
	(".xls",	"application/vnd.ms-excel"),
	(".y",		"text/plain; charset=utf-8"),
	(".zip",	"application/zip"),
};

Eok:			con 200;
Epartialcontent:	con 206;
Emovedpermanently:	con 301;
Enotmodified:		con 304;
Ebadrequest:		con 400;
Eunauthorized:		con 401;
Eforbidden:		con 403;
Enotfound:		con 404;
Emethodnotallowed:	con 405;
Elengthrequired:	con 411;
Epreconditionfailed:	con 412;
Ebadmediatype:		con 415;
Enotsatisfiable:	con 416;
Eexpectationfailed:	con 417;
Eservererror:		con 500;
Enotimplemented:	con 501;
Ebadversion:		con 505;

statusmsgs := array[] of {
	(100,		"Continue"),
	(200,		"OK"),
	(206,		"Partial Content"),
	(301,		"Moved Permanently"),
	(304,		"Not Modified"),
	(400,		"Bad Request"),
	(401,		"Unauthorized"),
	(403,		"Forbidden"),
	(404,		"Object Not Found"),
	(405,		"Method Not Allowed"),
	(411,		"Length Required"),
	(412,		"Precondition Failed"),
	(415,		"Unsupported Media Type"),
	(416,		"Requested Range Not Satisfiable"),
	(417,		"Expectation Failed"),
	(500,		"Internal Server Error"),
	(501,		"Not Implemented"),
	(505,		"HTTP Version Not Supported"),
};

# relevant known request headers whose values are not allowed to be concatenated (not a bnf #-rule, see rfc2616#2.1)
nomergeheaders := array[] of {
	# these two would be useful to merge.  alas, it is not allowed by rfc2616, section 4.2, last paragraph
	"if-match",
	"if-none-match",

	"authorization",
	"content-length",
	"content-type",
	"host",
	"if-modified-since",
	"if-range",
	"if-unmodified-since",
	"range",
};

idch: chan of int;
randch: chan of int;
killch: chan of int;
killschedch: chan of (int, int, chan of int);
excch: chan of (int, chan of string);
warnch: chan of (int, string);
cfgsgetch: chan of chan of ref Cfgs;
newcfgsch: chan of ref Cfgs;
logch: chan of (string, string);
logfdch: chan of ref Sys->FD;

timefd: ref Sys->FD;
errorfd: ref Sys->FD;

Cgi, Scgi: con iota;
cgitypes := array[] of {"cgi", "scgi"};

cgispawnch: chan of (string, string, string, ref Op, big, chan of (ref Sys->FD, ref Sys->FD, string));
scgidialch: chan of (string, chan of (ref Sys->FD, string));

init(nil: ref Draw->Context, args: list of string)
{
	sys = load Sys Sys->PATH;
	bufio = load Bufio Bufio->PATH;
	env = load Env Env->PATH;
	daytime = load Daytime Daytime->PATH;
	keyring = load Keyring Keyring->PATH;
	random = load Random Random->PATH;
	str = load String String->PATH;
	base64 = load Encoding Encoding->BASE64PATH;
	sh = load Sh Sh->PATH;
	ipm = load IP IP->PATH;
	ipm->init();
	attrdb = load Attrdb Attrdb->PATH;
	err := attrdb->init();
	if(err != nil)
		fail("loading attrdb: "+err);
	regex = load Regex Regex->PATH;
	http = load Http Http->PATH;
	http->init(bufio);

	cfgs: ref Cfgs;

	(cfgs, err) = Cfgs.init("/dev/null");
	if(err != nil)
		fail("making empty config: "+err);
	defcfg := cfgs.default;

	arg := load Arg Arg->PATH;
	arg->init(args);
	arg->setusage(arg->progname()+" [-dh] [-A path realm user:pass] [-C cachesecs] [-L listpath 0|1] [-a addr] [-c path command methods] [-f ctlchan] [-i indexfile] [-l logfile] [-n config] [-r pathre dest] [-s path addr methods] [-t extension mimetype] [-z gzippath] webroot");
	while((c := arg->opt()) != 0)
		case c {
		'A' =>	defcfg.auths = ref (arg->earg(), arg->earg(), base64->enc(array of byte arg->earg()))::defcfg.auths;
			if(haschar((hd defcfg.auths).t1, '"')) {
				fprint(fildes(2), "realm must not have double quote, not supported by http/1.0");
				raise "fail:usage";
			}
		'C' =>	defcfg.cachesecs = int arg->earg();
		'L' =>	path := arg->earg();
			what := int arg->earg();
			nl := array[len defcfg.listings+1] of ref (string, int);
			nl[:] = defcfg.listings;
			nl[len defcfg.listings] = ref (path, what);
			defcfg.listings = nl;
		'a' =>	cfgs.addrs = arg->earg()::cfgs.addrs;
		'c' =>	defcfg.cgipaths = ref (arg->earg(), arg->earg(), sys->tokenize(arg->earg(), " ,").t1, Cgi)::defcfg.cgipaths;
		'd' =>	cfgs.debugflag++;
		'f' =>	ctlchan = arg->earg();
		'h' =>	cfgs.vhostflag++;
		'i' =>	defcfg.indexfiles = arg->earg()::defcfg.indexfiles;
		'l' =>	logfile := arg->earg();
			cfgs.accessfd = sys->open(logfile, Sys->OWRITE);
			if(cfgs.accessfd == nil)
				fail(sprint("open logfile %q: %r", logfile));
			sys->seek(cfgs.accessfd, big 0, Sys->SEEKEND);

		'n' =>
			file := arg->earg();
			(cfgs, err) = Cfgs.init(file);
			if(err != nil) {
				fprint(fildes(2), "reading %q: %s\n", file, err);
				raise "fail:usage";
			}
			defcfg = cfgs.default;
			defcfg.rev();
			cfgs.usertypes = rev(cfgs.usertypes);
			cfgs.addrs = rev(cfgs.addrs);
		'r' =>
			(restr, rulestr) := (arg->earg(), arg->earg());
			(repl, rerr) := Repl.parse(restr, rulestr);
			if(err != nil) {
				fprint(fildes(2), "parsing redir %q %q: %s\n", restr, rulestr, rerr);
				raise "fail:usage";
			}
			defcfg.redirs = repl::defcfg.redirs;
		's' =>	defcfg.cgipaths = ref (arg->earg(), arg->earg(), sys->tokenize(arg->earg(), " ,").t1, Scgi)::defcfg.cgipaths;
		't' =>	cfgs.usertypes = ref (arg->earg(), arg->earg())::cfgs.usertypes;
		'z' =>	defcfg.gzpaths = arg->earg()::defcfg.gzpaths;
		* =>	arg->usage();
		}
	args = arg->argv();
	if(len args != 1)
		arg->usage();
	webroot = hd args;
	defcfg.rev();
	cfgs.usertypes = rev(cfgs.usertypes);
	cfgs.addrs = rev(cfgs.addrs);
	credempty = base64->enc(array of byte ":");	# empty-user:empty-pass

	environment = env->getall();

	pid := sys->pctl(Sys->NEWPGRP|Sys->FORKNS|Sys->FORKENV|Sys->FORKFD, nil);
	if(pid < 0)
		fail(sprint("pctl: %r"));
	if(sys->chdir(webroot) != 0)
		fail(sprint("chdir webroot %q: %r", webroot));

	if(ctlchan != nil) {
		fio := sys->file2chan("/chan", ctlchan);
		if(fio == nil)
			fail(sprint("file2chan in /chan: %q: %r", ctlchan));
		spawn ctlhandler(fio);
	}

	timefd = sys->open("/dev/time", Sys->OREAD);
	if(timefd == nil)
		fail(sprint("open /dev/time: %r"));

	errorfd = sys->open("/services/logs/shttpderror", Sys->OWRITE);
	if(errorfd != nil)
		sys->seek(errorfd, big 0, Sys->SEEKEND);

	idch = chan[8] of int;
	spawn idgen();
	randch = chan of int;
	spawn randgen();
	killch = chan of int;
	killschedch = chan of (int, int, chan of int);
	spawn killer();
	excch = chan of (int, chan of string);
	spawn exceptsetter();
	warnch = chan of (int, string);
	spawn warner();

	cfgsgetch = chan of chan of ref Cfgs;
	newcfgsch = chan of ref Cfgs;
	spawn cfgsserver();

	logch = chan of (string, string);
	logfdch = chan of ref Sys->FD;
	spawn logger();

	newcfgsch <-= cfgs;

	cgispawnch = chan of (string, string, string, ref Op, big, chan of (ref Sys->FD, ref Sys->FD, string));
	spawn cgispawner();

	scgidialch = chan of (string, chan of (ref Sys->FD, string));
	spawn scgidialer();

	if(cfgs.addrs == nil)
		cfgs.addrs = defaddr::nil;
	for(addrs := rev(cfgs.addrs); addrs != nil; addrs = tl addrs) {
		addr := hd addrs;
		(aok, aconn) := sys->announce(addr);
		if(aok != 0)
			fail(sprint("announce %q: %r", addr));
		spawn listen(hd addrs, aconn, sync := chan of int);
		<-sync;
	}
	warn(0, sprint("shttpd started at %s", daytime->time()));
}

listen(addr: string, aconn: Sys->Connection, sync: chan of int)
{
	sync <-= 0;
	for(;;) {
		(lok, lconn) := sys->listen(aconn);
		if(lok != 0)
			fail(sprint("listen %q: %r", addr));
		dfd := sys->open(lconn.dir+"/data", Sys->ORDWR);
		if(dfd != nil)
			spawn httpserve(dfd, lconn.dir);
		else
			warn(0, sprint("open connection file: %r"));
		lconn.dfd = nil;
	}
}

idgen()
{
	id := 1;
	for(;;)
		idch <-= id++;
}

randgen()
{
	for(;;)
		randch <-= random->randomint(Random->NotQuiteRandom);
}

killer()
{
	for(;;) alt {
	pid := <-killch =>
		kill(pid);
	(pid, timeout, respch) := <-killschedch =>
		spawn timeoutkill(pid, timeout, respch);
	}
}

cfgsserver()
{
	cfgs: ref Cfgs;

	for(;;) alt {
	respch := <-cfgsgetch =>
		respch <-= cfgs;
	cfgs = <-newcfgsch =>
		# non-atomic wrt other threads...  not much to do about it
		debugflag = cfgs.debugflag;
		logfdch <-= cfgs.accessfd;
	}
}

timeoutkill(pid, timeout: int, respch: chan of int)
{
	respch <-= sys->pctl(0, nil);
	sys->sleep(timeout);
	kill(pid);
}

exceptsetter()
{
	for(;;) {
		(pid, respch) := <-excch;
		err: string;
		fd := sys->open(sprint("/prog/%d/ctl", pid), Sys->OWRITE);
		if(fd == nil || fprint(fd, "exceptions notifyleader") == -1)
			err = sprint("setting exception handling for pid %d: %r", pid);
		if(respch == nil && err != nil) {
			warn(0, sprint("setting exceptions notifyleader for pid %d: %s", pid, err));
			kill(pid);
		}
		if(respch != nil)
			respch <-= err;
	}
}

warner()
{
	for(;;) {
		(id, s) := <-warnch;
		if(s != nil && s[len s-1] == '\n')
			s = s[:len s-1];
		warn(id, s);
	}
}

cgispawner()
{
	for(;;) {
		(cmd, path, cgipath, op, length, replych) := <-cgispawnch;
		spawn cgispawn(cmd, path, cgipath, op, length, replych);
	}
}

cgispawn(cmd, path, cgipath: string, op: ref Op, length: big, replych: chan of (ref Sys->FD, ref Sys->FD, string))
{
	p0 := array[2] of ref Sys->FD;
	p1 := array[2] of ref Sys->FD;
	p2 := array[2] of ref Sys->FD;
	if(sys->pipe(p0) != 0 || sys->pipe(p1) != 0 || sys->pipe(p2) != 0) {
		replych <-= (nil, nil, sprint("pipe: %r"));
		return;
	}

	spawn errlogger(op, p2[0]);

	if(sys->pctl(Sys->NEWPGRP|Sys->NEWFD|Sys->FORKNS|Sys->FORKENV, p0[0].fd::p1[0].fd::p0[1].fd::p1[1].fd::p2[1].fd::nil) < 0) {
		replych <-= (nil, nil, sprint("pctl newpgrp,newfd,forkns,forkenv: %r"));
		return;
	}

	for(l := cgivars(path, cgipath, op, length, nil); l != nil; l = tl l) {
		epath := "/env/"+(hd l).t0;
		efd := sys->create(epath, Sys->OWRITE, 8r666);
		if(efd == nil) {
			replych <-= (nil, nil, sprint("open %q: %r", epath));
			return;
		}
		d := array of byte (hd l).t1;
		if(sys->write(efd, d, len d) != len d) {
			replych <-= (nil, nil, sprint("write %q: %r", epath));
			return;
		}
	}

	if(sys->dup(p0[1].fd, 0) == -1 || sys->dup(p1[1].fd, 1) == -1 || sys->dup(p2[1].fd, 2) == -1) {
		replych <-= (nil, nil, sprint("dup: %r"));
		return;
	}

	replych <-= (p0[0], p1[0], nil);

	p0[1] = fildes(p0[1].fd);
	p1[1] = fildes(p1[1].fd);
	p2[1] = fildes(p2[1].fd);
	p0[0] = p1[0] = p2[0] = nil;

	err := sh->system(nil, cmd);
	if(err != nil)
		warnch <-= (op.id, sprint("cgispawn, command %q: %s", cmd, err));
}

errlogger(op: ref Op, fd: ref Sys->FD)
{
	for(;;) {
		n := sys->read(fd, d := array[Sys->ATOMICIO] of byte, len d);
		if(n < 0)
			warnch <-= (op.id, sprint("reading stderr: %r"));
		if(n <= 0)
			break;
		warnch <-= (op.id, string d[:n]);
	}
}

scgidialer()
{
	for(;;) {
		(scgiaddr, replychan) := <-scgidialch;
		spawn scgidial(scgiaddr, replychan);
	}
}

scgidial(scgiaddr: string, replychan: chan of (ref Sys->FD, string))
{
	(ok, conn) := sys->dial(scgiaddr, nil);
	if(ok < 0)
		replychan <-= (nil, sprint("dialing scgid %q: %r", scgiaddr));
	else
		replychan <-= (conn.dfd, nil);
}

ctlhandler(fio: ref Sys->FileIO)
{
	for(;;) alt {
	(nil, nil, nil, rc) := <- fio.read =>
		if(rc == nil)
			continue;
		rc <-= (nil, "permission denied");

        (nil, data, nil, wc) := <- fio.write =>
                if(wc == nil)
                        continue;
		s := string data;
		if(s != nil && s[len s-1] == '\n')
			s = s[:len s-1];
		case s {
		"reload" =>
			cfgsgetch <-= respch := chan of ref Cfgs;
			cfgs := <-respch;

			if(cfgs.db.reopen() != 0) {
				msg := sprint("reopening config file: %r");
				warn(0, msg);
				wc <-= (0, msg);
				continue;
			}
			(ncfgs, err) := Cfgs.read(cfgs.file, cfgs.db);
			if(err != nil) {
				msg := "error reloading config, keeping current: "+err;
				warn(0, msg);
				wc <-= (0, msg);
				continue;
			}
			newcfgsch <-= ncfgs;
			warn(0, "config file reloaded");
			wc <-= (len data, nil);
		* =>
			wc <-= (0, sprint("bad command: %q", s));
		}
	}
}

httpserve(fd: ref Sys->FD, conndir: string)
{
	id := <-idch;
	if(debugflag) say(id, "httpserve");

	(lhost, lport) := readaddr(id, conndir+"/local");
	(rhost, rport) := readaddr(id, conndir+"/remote");
	lhost = IPaddr.parse(lhost).t1.text();
	rhost = IPaddr.parse(rhost).t1.text();
	if(debugflag) say(id, sprint("connect from %s:%s to %s:%s", rhost, rport, lhost, lport));

	pid := sys->pctl(Sys->NEWPGRP|Sys->FORKNS|Sys->NODEVS, nil);
	if(pid < 0)
		die(id, sprint("pctl: %r"));
	excch <-= (pid, nil);
	if(sys->bind(webroot,  "/", Sys->MREPL) < 0)
		die(id, sprint("bind %q /: %r", webroot));

	b := bufio->fopen(fd, Bufio->OREAD);
	if(b == nil)
		die(id, sprint("bufio open: %r"));

	op := ref Op(id, 0, 0, 0, big 0, fd, b, rhost, rport, lhost, lport, nil, nil, nil, nil);

	for(nsrvs := 0; ; nsrvs++) {
		if(nsrvs > 0 && !op.keepalive)
			break;

		if(sys->chdir("/") != 0)
			break;

		op.chunked = op.keepalive = 0;
		op.length = big -1;
		op.req = nil;
		op.resp = nil;
		op.cfgs = nil;
		op.cfg = nil;
		httptransact(pid, b, op);
	}
}

httptransact(pid: int, b: ref Iobuf, op: ref Op)
{
	id := op.id;
	op.now = readtime();

	hdrs := Hdrs.new(("server", Version)::nil);

	# "You know they'll never really die while the Trunk is alive[...]
	#  It lives while the code is shifted, and they live with it, always Going Home."
	# - Moist von Lipwig, Going Postal, Chapter 13
	# "A man is not dead while his name is still spoken."
	# - Going Postal, Chapter 4 prologue
	# <http://www.gnuterrypratchett.com/>
	hdrs.add("X-Clacks-Overhead", "GNU Geo Rivera");
	hdrs.add("X-Clacks-Overhead", "GNU Alejandra Agredo");
	# we love and miss you dear friends

	# disable FLoC
	hdrs.add("Permissions-Policy", "interest-cohort=()");

	# kill ourself when no request comes in
	killschedch <-= (pid, Keepalivesecs*1000, respch := chan of int);
	killpid := <-respch;

	(req, rerr) := Req.read(b);
	cfgsgetch <-= cfgsrespch := chan of ref Cfgs;
	op.cfgs = <-cfgsrespch;

	hdrs.add("date", httpdate(op.now));
	if(rerr != nil || req.major != 1 || req.method == UNKNOWN) {
		st := Ebadrequest;
		if(rerr == nil && req.major != 1) {
			st = Ebadversion;
			rerr = sprint("Version requested is HTTP/%d.%d", req.major, req.minor);
		} else if(rerr == nil && req.method == UNKNOWN) {
			st = Enotimplemented;
			rerr = sprint("Method %q not supported", req.methodstr);
		}

		stmsg := statusmsg(st);
		op.resp = Resp.mk(HTTP_10, string st, stmsg, hdrs);

		html := array of byte mkhtml(sprint("%d - %s: %s", st, stmsg, rerr));

		op.resp.h.add("content-type", "text/html; charset=utf-8");
		op.resp.h.add("content-length", string len html);

		err := hresp(op.resp, op.fd, 0, 0);
		if(err == nil)
			sys->write(op.fd, html, len html);

		killch <-= killpid;
		die(id, "reading request: "+rerr);
	}
	killch <-= killpid;
	if(debugflag) say(id, sprint("request: method %q url %q version %q",
		req.methodstr, req.url.pack(), sprint("HTTP/%d.%d", req.major, req.minor)));
	op.req = req;

	# all values besides "close" are supposedly header names, not important
	(contoks, conerr) := tokenize(req.h.getlist("connection"));
	if(conerr != nil || len contoks == 0 && req.h.has("connection", nil))
		return responderrmsg(op, Ebadrequest, sprint("Bad Request: Bad value for header \"Connection\""));

	op.keepalive = req.version() >= HTTP_11 && conerr == nil && !listhas(contoks, "close");
	op.resp = resp := Resp.mk(req.version(), "200", "OK", hdrs);

	# tell client if it is sending ambiguous requests: duplicate headers of the important kind
	for(i := 0; i < len nomergeheaders; i++)
		if(len req.h.findall(nomergeheaders[i]) > 1)
			return responderrmsg(op, Ebadrequest, sprint("Bad Request: Duplicate headers:  \"%s\"", nomergeheaders[i]));

	if(req.h.has("proxy-authorization", nil))
		return responderrmsg(op, Ebadrequest, "Bad Request: Proxy-Authorization credentials sent, unacceptable");

	if(req.version() >= HTTP_11 && !req.h.has("host", nil))
		return responderrmsg(op, Ebadrequest, "Bad Request: Missing header \"Host\"");

	# when host-header is absent, we'll request the empty host name, the default
	host := splithost(req.h.get("host")).t0;
	if(str->drop(host, "0-9a-zA-Z.:-") != nil || str->splitstrl(host, "..").t1 != nil)
		return responderrmsg(op, Ebadrequest, nil);

	cfg := op.cfg = op.cfgs.find(host, op.lport);
	if(cfg == nil)
		return responderrmsg(op, Enotfound, nil);

	# do not accept request when doing vhost and request is from ip that we shouldn't serve host:port on
	if(op.cfgs.vhostflag && cfg.listens != nil) {
		addrokay := 0;
		for(ls := cfg.listens; !addrokay && ls != nil; ls = tl ls) {
			(chost, cport) := *(hd ls);
			addrokay = chost == op.lhost && cport == op.lport;
		}
		if(!addrokay) {
			say(id, "request on ip:port, not allowed");
			return responderrmsg(op, Enotfound, nil);
		}
	}

	if(req.version() == HTTP_10 && req.method != GET && req.method != HEAD && req.method != POST)
		return responderrmsg(op, Enotimplemented, sprint("Unknown Method: \"%s\"", req.methodstr));

	if(hasbody(op.req) && (req.method == GET || req.method == HEAD || req.method == TRACE || req.method == DELETE)) {
		op.keepalive = 0;
		return responderrmsg(op, Ebadrequest, "Bad Request: Entity not allowed in request");
	}
	# for other methods, we ignore bodies by closing the connection.  saner than reading and discarding...

	case req.method {
	GET or HEAD =>
		;
	POST =>
		if(hasbody(op.req))
			op.keepalive = 0;
	TRACE =>
		# bug: the response does not have * as path, but /
		return respond(op, Eok, req.pack(), "message/http");

	OPTIONS =>
		# only (s)cgi paths allow POST, but we won't say, the path may require auth as well.  what to do then?
		hdrs.add("allow", "OPTIONS, GET, HEAD, POST, TRACE");
		hdrs.add("accept-ranges", "bytes");
		if(hasbody(op.req))
			op.keepalive = 0;
		return responderrmsg(op, Eok, nil);

	PUT or DELETE =>
		# note: when implementing these, complete support for if-match and if-none-match, and much more probably
		if(hasbody(op.req))
			op.keepalive = 0;
		return responderrmsg(op, Enotimplemented, "Not Implemented: PUT and DELETE are not supported");

	* =>
		if(hasbody(op.req))
			op.keepalive = 0;
		return responderrmsg(op, Enotimplemented, sprint("Unknown Method: \"%s\"", req.methodstr));
	}

	# remove occurrences of "/elem/../" from path, returned path always starts with "/"
	path := pathsanitize(req.url.path);

	# we ignore the port in the host-header.  this is illegal according to rfc2616, but using it is just silly.
	# also, we violate rfc2616 by sending 404 "not found" when the host doesn't exist.
	# we should send 400 "bad request" then, but that is just silly too.
	if(op.cfgs.vhostflag) {
		hostdir: string;
		if(cfg.host != "") {
			hostdir = cfg.host+"!"+cfg.port;
			if(sys->chdir(hostdir) != 0) {
				if(debugflag) say(id, sprint("using hostdir %q from config failed, trying default", hostdir));
				hostdir = nil;
			}
		}
		if(hostdir == nil) {
			hostdir = "_default!"+cfg.port;
			if(sys->chdir(hostdir) != 0)
				return responderrmsg(op, Enotfound, nil);
		}
		if(debugflag) say(id, sprint("using hostdir %q, path %q", hostdir, path));
	}

	validauth := needauth := 0;
	realm: string;
	authtype, cred: string;
	(authtype, cred) = str->splitstrr(req.h.get("authorization"), " ");
	if(str->tolower(authtype) != "basic ")
		cred = nil;
	else
		cred = stripws(cred);
	for(a := cfg.auths; !validauth && a != nil; a = tl a) {
		(apath, arealm, acred) := *hd a;
		if(str->prefix(apath, path)) {
			needauth = 1;
			realm = arealm;
			validauth = cred == acred;
		}
	}
	if(needauth && !validauth) {
		resp.h.add("www-authenticate", sprint("Basic realm=\"%s\"", realm));
		return responderrmsg(op, Eunauthorized, nil);
	}
	if(req.h.has("authorization", nil) && !needauth && cred != credempty) {
		resp.h.add("www-authenticate", sprint("Basic realm=\"authentication not allowed, use empty username/password\""));
		return responderrmsg(op, Eunauthorized, "Not Authorized:  Sending authorization credentials is not allowed for "+
			"this resource.  Please use an empty username and password or do not send authorization credentials altogether.");
	}
	if(debugflag && validauth) say(id, "have valid auth credentials");

	for(r := cfg.redirs; r != nil; r = tl r) {
		repl := hd r;
		(match, dest, replerr) := repl.apply(path);
		if(replerr != nil) {
			warn(id, "redirections misconfiguration: "+replerr);
			return responderrmsg(op, Eservererror, "Internal Server Error: An error occurred while handling a redirection");
		}
		if(!match)
			continue;
		if(!str->prefix("http://", dest) && !str->prefix("https://", dest)) {
			(havehost, hosthdr) := req.h.find("host");
			if(havehost) {
				dest = "http://"+hosthdr+dest;
			} else {
				lport := "";
				if(op.lport != "80")
					lport = ":"+op.lport;
				httphost := op.lhost;
				(ok, ip) := IPaddr.parse(httphost);
				if(ok == 0 && !ip.isv4())
					httphost = "["+httphost+"]";
				dest = "http://"+httphost+lport+dest;
			}
		}
		if(debugflag) say(id, sprint("redirecting from %q to %q", path, dest));
		resp.h.set("location", dest);
		dest = htmlescape(dest);
		st := Emovedpermanently;
		html := mkhtmlstart(sprint("%d - %s", st, statusmsg(st))) +
			sprint("<h1>Moved Permanently: Moved to <a href=\"%s\">%s</a></h1>\n", dest, dest)+"</body></html>\n";
		return respond(op, Emovedpermanently, html, "text/html; charset=utf-8");
	}

	if(((cgipath, cgiaction, methods, cgitype) := findcgi(cfg, path)).t1 != nil) {
		if(methods != nil && !hasmethod(methods, req.methodstr))
			return responderrmsg(op, Emethodnotallowed, nil);
		return cgi(path, op, cgipath, cgiaction, cgitype);
	}

	# path is one of:  plain file, directory (either listing or plain index file)
	dfd := sys->open("."+path, Sys->OREAD);
	if(dfd != nil)
		(dok, dir) := sys->fstat(dfd);
	if(dok == 0 && dir.mode&Sys->DMDIR && path[len path-1] == '/') {
		for(l := cfg.indexfiles; l != nil; l = tl l) {
			ipath := "."+path+hd l;
			(iok, idir) := sys->stat(ipath);
			if(iok != 0 || idir.mode&Sys->DMDIR)
				continue;
			ifd := sys->open(ipath, Sys->OREAD);
			if(ifd == nil)
				return responderrmsg(op, Enotfound, nil);
			if(debugflag) say(id, sprint("using index file %q", hd l));
			dfd = ifd;
			dir = idir;
			path += hd l;
			break;
		}
	}
	if(dfd == nil || dok != 0 || (dir.mode&Sys->DMDIR) && (!dolisting(cfg, path) || path != nil && path[len path-1] != '/'))
		return responderrmsg(op, Enotfound, nil);

	if(req.method == POST) {
		resp.h.add("allow", "GET, HEAD, OPTIONS");
		return responderrmsg(op, Emethodnotallowed, "Method Not Allowed: POST not allowed");
	}

	resp.h.add("last-modified", httpdate(dir.mtime));
	tag := etag(path, op, dir);
	resp.h.add("etag", tag);

	ifmatch, ifnonematch, ifunmodsincestr: string;
	havecond: int;
	(havecond, ifmatch) = req.h.find("if-match");
	if(req.version() >= HTTP_11 && havecond && !etagmatch(req.version(), tag, ifmatch, 1))
		return responderrmsg(op, Epreconditionfailed, sprint("Precondition Failed: Etag(s) \"%s\", specified with If-Match did not match", ifmatch));

	ifmodsince := parsehttpdate(req.h.get("if-modified-since"));
	# http/1.0, head and if-modified-since: rfc1945#8.1;  unsupported date value can safely be ignored.
	if(!(req.version() == HTTP_10 && req.method == HEAD) && ifmodsince && dir.mtime <= ifmodsince)
		return responderr(op, Enotmodified);

	# note: for get this is okay, but for put/delete a bad-syntax value would be have to raise an error
	(havecond, ifnonematch) = req.h.find("if-none-match");
	if(req.version() >= HTTP_11 && havecond && req.method == GET && etagmatch(req.version(), tag, ifnonematch, 0))
		return responderr(op, Enotmodified);

	# unsupported date value causes a "precondition failed"
	(havecond, ifunmodsincestr) = req.h.find("if-unmodified-since");
	ifunmodsince := parsehttpdate(ifunmodsincestr);
	if(req.version() >= HTTP_11 && (ifunmodsince && dir.mtime > ifunmodsince || havecond && ifunmodsince == 0))
		return responderrmsg(op, Epreconditionfailed, sprint("Precondition Failed: Object has been modified since \"%s\"", req.h.get("if-unmodified-since")));

	if(dir.mode&Sys->DMDIR)
		listdir(path, op, dfd);
	else
		plainfile(path, op, dfd, dir, tag);
}

pathsanitize(path: string): string
{
	trailslash := path != nil && path[len path-1] == '/';

	(nil, elems) := sys->tokenize(path, "/");
	r: list of string;
	for(; elems != nil; elems = tl elems)
		if(hd elems == ".")
			continue;
		else if(hd elems == "..") {
			if(r != nil)
				r = tl r;
		} else
			r = hd elems::r;
	s := "";	
	for(; r != nil; r = tl r)
		s = "/"+hd r+s;
	if(trailslash || s == "")
		s += "/";
	return s;
}

findcgi(cfg: ref Cfg, path: string): (string, string, list of string, int)
{
	for(l := cfg.cgipaths; l != nil; l = tl l)
		if(str->prefix((*hd l).t0, path))
			return *hd l;
	return (nil, nil, nil, 0);
}

etag(path: string, op: ref Op, dir: Sys->Dir): string
{
	host := splithost(op.req.h.get("host")).t0;
	if(host == nil)
		host = "_default";
	return "\""+sha1(array of byte sprint("%d,%d,%s,%s,%s", dir.qid.vers, dir.mtime, host, op.lport, path))+"\"";
}

etagmatch(version: int, etag: string, etagstr: string, strong: int): int
{
	if(etagstr == "*")
		return 1;
	(l, err) := tokenizeqs(etagstr, version);
	if(err != nil)
		return 0;
	for(; l != nil; l = tl l)
		if(hd l == etag && (!strong || !str->prefix("W/", hd l)))
			return 1;
	return 0;
}

plainfile(path: string, op: ref Op, dfd: ref Sys->FD, dir: Sys->Dir, tag: string)
{
	id := op.id;
	req := op.req;
	resp := op.resp;

	if(debugflag) say(id, "doing plain file");
	ct := mimetype(op.cfgs, path);
	if(ct != nil)
		resp.h.add("content-type", ct);
	else
		warn(op.id, sprint("could not determine content-type:  host %q path %q query %q", op.req.h.get("host"), op.req.url.path, op.req.url.query));

	# synthetic files sometimes don't have meaningful dir.length.
	# there is no one true way to determine if a file is synthetic.  this is the most reasonable.
	issynthetic := dir.length == big 0 && dir.qid.vers == 0;
	if(issynthetic) {
		op.chunked = resp.version() >= HTTP_11;
	} else {
		op.length = dir.length;
		resp.h.add("content-length", string op.length);
	}

	if(issynthetic && (req.h.has("range", nil) || req.h.has("if-range", nil)))
		return responderrmsg(op, Enotsatisfiable, nil);

	(valid, ranges) := parserange(req.version(), req.h.find("range"), dir.length);
	if(!valid) {
		resp.h.add("content-range", sprint("bytes */%bd", dir.length));
		return responderrmsg(op, Enotsatisfiable, nil);
	}
	bound := "";
	ifrange := req.h.get("if-range");
	# unsupported date value can safely be ignored.
	if(ranges != nil && (ifrange == nil
	                     || ifrange[0] == '"' && tag == ifrange
	                     || dir.mtime <= parsehttpdate(ifrange))) {
		if(len ranges == 1) {
			(start, end) := *hd ranges;
			resp.h.add("content-range", sprint("bytes %bd-%bd/%bd", start, end-big 1, dir.length));
			resp.h.set("content-length", string (end-start));
		} else {
			bound = sha1(array of byte (string <-randch+","+string op.now));
			resp.h.set("content-type", "multipart/byteranges; boundary="+bound);
			resp.h.del("content-length", nil);
			op.chunked = resp.version() >= HTTP_11;
		}
		resp.st = string Epartialcontent;
		resp.stmsg = "Partial Content";
		if(debugflag) say(id, sprint("sending %d ranges for multipart/byterange response", len ranges));
	} else
		ranges = ref (big 0, dir.length)::nil;

	if(gzipallowed(op.cfg.gzpaths, path))
	if(gzipaccepted(req.h))
	if(!issynthetic)
	if(!req.h.find("range").t0)
	if(!req.h.find("if-range").t0)
	if((gzfd := sys->open(gzpath := "."+path+".gz", sys->OREAD)) != nil)
	if(((nil, gzd) := sys->stat(gzpath)).t0 == 0)
	if(gzd.length < dir.length)
	if(gzd.mtime >= dir.mtime) {
		op.length = gzd.length;
		ranges = ref (big 0, gzd.length)::nil;
		resp.h.add("content-encoding", "gzip");
		resp.h.set("content-length", string gzd.length);
		dfd = gzfd;
		dir = gzd;
	}

	if(op.cfg.cachesecs >= 0)
		resp.h.add("cache-control", sprint("max-age=%d", op.cfg.cachesecs));

	accesslog(op);

	rerr := hresp(resp, op.fd, op.keepalive, op.chunked);
	if(rerr != nil)
		die(id, "writing response: "+rerr);

	if(req.method == HEAD)
		return;

	if(issynthetic) {
		buf := array[Sys->ATOMICIO] of byte;
		for(;;) {
			n := sys->readn(dfd, buf, len buf);
			if(n == 0)
				break;
			if(n < 0)
				die(id, sprint("reading file: %r"));
			hwrite(op, buf[:n]);
		}
		hwriteeof(op);
		return;
	}

	for(; ranges != nil; ranges = tl ranges) {
		(off, end) := *hd ranges;
		if(bound != nil) {
			s := sprint("--%s\r\ncontent-type: %s\r\ncontent-range: bytes %bd-%bd/%bd\r\n\r\n",
				bound, ct, off, end-big 1, dir.length);
			hwrite(op, array of byte s);
		}
		while(off < end) {
			want := end-off;
			if(want > big Sys->ATOMICIO)
				want = big Sys->ATOMICIO;
			n := sys->pread(dfd, d := array[int want] of byte, len d, off);
			if(n < 0)
				die(id, sprint("reading file: %r"));
			if(n == 0)
				break;
			off += big n;
			hwrite(op, d[:n]);
		}
		if(bound != nil)
			hwrite(op, array of byte "\r\n");
	}
	hwriteeof(op);
}

gzipallowed(l: list of string, p: string): int
{
	for(; l != nil; l = tl l)
		if(str->prefix(hd l, p))
			return 1;
	return 0;
}

# warning: parsing is simplistic, not handling full http rules, but should be good enough for finding "gzip".
gzipaccepted(h: ref Hdrs): int
{
	for(l := sys->tokenize(h.get("accept-encoding"), ",").t1; l != nil; l = tl l)
		if(strip(str->splitstrl(hd l, ";").t0, " \t") == "gzip")
			return 1;
	return 0;
}

dolisting(cfg: ref Cfg, path: string): int
{
	for(i := 0; i < len cfg.listings; i++) {
		(lpath, what) := *cfg.listings[i];
		if(str->prefix(lpath, path)) {
			say(-1, sprint("listing for path %q: %d, prefix: %q", path, what, lpath));
			return what;
		}
	}
	return 0;
}

listdir(path: string, op: ref Op, dfd: ref Sys->FD)
{
	id := op.id;
	req := op.req;
	resp := op.resp;

	if(debugflag) say(id, "doing directory listing");
	resp.h.add("content-type", "text/html; charset=utf-8");
	op.chunked = resp.version() >= HTTP_11;

	if(op.cfg.cachesecs >= 0)
		resp.h.add("cache-control", sprint("max-age=%d", op.cfg.cachesecs));

	accesslog(op);

	rerr := hresp(resp, op.fd, op.keepalive, op.chunked);
	if(rerr != nil)
		die(id, "writing response: "+rerr);

	if(req.method == HEAD)
		return;

	begin := mkhtmlstart("listing for "+path) + sprint("<h1>listing for %s</h1><hr/><table><tr><th>last modified</th><th>size</th><th>name</th></tr>\n", pathurls(path));
	hwrite(op, array of byte begin);
	for(;;) {
		(nd, d) := sys->dirread(dfd);
		if(nd < 0)
			die(id, sprint("reading dir: %r"));
		if(nd == 0)
			break;
		html := "";
		for(i := 0; i < nd && i < len d; i++) {
			name := d[i].name;
			if(d[i].mode & Sys->DMDIR)
				name += "/";
			html += sprint("<tr><td class=\"mtime\">%s</td><td class=\"size\">%bd</td><td class=\"name\"><a href=\"%s\">%s</a></td></tr>\n", daytime->filet(op.now, d[i].mtime), d[i].length, htmlescape(encodepath("./"+name)), htmlescape(name));
		}
		hwrite(op, array of byte html);
	}
	end := sprint("</table><hr/></body></html>\n");
	hwrite(op, array of byte end);
	hwriteeof(op);
}

mkhtmlstart(msg: string): string
{
	return sprint("<html><head>\n<style type=\"text/css\">\nh1 { font-size: 1.4em; }\ntd, th { padding-left: 1em; padding-right: 1em; }\ntd.mtime, td.size { text-align: right; }\n</style>\n<title>%s</title>\n</head><body>\n", htmlescape(msg));
}

mkhtml(msg: string): string
{
	return mkhtmlstart(msg)+sprint("\n<h1>%s</h1>\n</body></html>\n", htmlescape(msg));
}

htmlescape(s: string): string
{
	r := "";
	for(i := 0; i < len s; i++)
		case s[i] {
		'<' =>	r += "&lt;";
		'>' =>	r += "&gt;";
		'&' =>	r += "&amp;";
		'"' =>	r += "&quot;";
		* =>	r += s[i:i+1];
		}
	return r;
}

pathurls(s: string): string
{
	(nil, l) := sys->tokenize(s, "/");
	r := "";
	i := 0;
	path := "./";
	for(l = rev(l); l != nil; l = tl l) {
		r = sprint(" <a href=\"%s\">%s/</a>", path, htmlescape(hd l))+r;
		if(i == 0)
			path = "../";
		else
			path += "../";
		i += 1;
	}
	r = sprint("<a href=\"%s\">/</a>", path)+r;
	return r;
}

cgi(path: string, op: ref Op, cgipath, cgiaction: string, cgitype: int)
{
	id := op.id;
	req := op.req;

	if(debugflag) say(id, sprint("passing to (s)cgi handler, cgipath %q cgiaction %q", cgipath, cgiaction));

	# first, some sanity checks on the request
	# parsing/handling full transfer-coding is too involved for us.
	# we are taking a short cut here to avoid feeding the bloat monster.
	length := big 0;
	needcontinue := 0;
	if(req.method == POST) {
		transferenc := req.h.getlist("transfer-encoding");
		if(req.version() >= HTTP_11 && transferenc != nil && transferenc != "identity")
			return responderrmsg(op, Enotimplemented, "Not Implemented: Transfer-Encodings other than \"identity\" "+
				"(i.e. no transfer encoding) are not supported");

		if(req.h.has("content-length", nil)) {
			lengthstr := req.h.get("content-length");
			if(lengthstr == nil || str->drop(lengthstr, "0-9") != "")
				return responderrmsg(op, Ebadrequest,
					sprint("Bad Request: Invalid Content-Length: \"%s\"", lengthstr));
			length = big lengthstr;
		} else {
			(e, emsg) := (Elengthrequired, "");
			if(req.version() == HTTP_10) # rfc1945#7.2.2
				(e, emsg) = (Ebadrequest, "Bad Request: Missing header \"Content-Length\"");
			return responderrmsg(op, e, emsg);
		}

		contentenc := req.h.getlist("content-encoding");
		if(contentenc != nil && contentenc != "identity")
			return responderrmsg(op, Enotimplemented, "Not Implemented: Content-Encoding other than identity "+
				"(i.e. no content encoding) are not supported");

		needcontinue = req.version() >= HTTP_11 && req.h.has("expect", nil);
		if(needcontinue && (expect := str->tolower(req.h.getlist("expect"))) != "100-continue")
			return responderrmsg(op, Eexpectationfailed,
				sprint("Expectectation Failed: Unrecognized expectation:  %s", expect));

		if(debugflag) say(id, sprint("post, client content-length %bd", length));
	}

	timeo := Cgitimeoutsecs*1000;
	donech := chan of int;

	spawn timeout(op, timeo, timeoch := chan of int, donech);
	timeopid := <- timeoch;
	if(timeopid < 0)
		die(op.id, "timeout proc failed");
	spawn cgiproc(path, op, cgipath, cgiaction, cgitype, length, needcontinue, timeopid, timeoch, donech);

	# wait for timeout or cgiproc
	<-donech;
	return;
}

timeout(op: ref Op, timeo: int, timeoch, donech: chan of int)
{
	pid := sys->pctl(Sys->NEWPGRP, nil);
	timeoch <-= pid;
	if(pid < 0)
		return warn(op.id, sprint("pctl: %r"));
		
	opid := <-timeoch;
	sys->sleep(timeo);
	if(debugflag) say(op.id, sprint("timeout %d ms for request, killing handler pid %d, timeopid %d", timeo, opid, pid));
	killch <-= opid;
	responderrmsg(op, Eservererror, "Internal Server Error: Response could not be generated in time");
	donech <-= 0;
}

cgiproc(path: string, op: ref Op, cgipath, cgiaction: string, cgitype: int, length: big, needcontinue: int, timeopid: int, cgich, donech: chan of int)
{
	# set up new process group and exception propagation so
	# we always clean up nicely when one of the child procs dies.
	# we always have to respond on donech (or be killed by the timeout
	# proc) or we'll leave processes lingering
	err: string;
	npid := sys->pctl(Sys->NEWPGRP, nil);
	if(npid < 0)
		err = sprint("pctl newpgrp: %r");
	if(err == nil) {
		excch <-= (npid, respch := chan of string);
		err = <-respch;
	}

	if(err != nil) {
		killch <-= timeopid;
		warn(op.id, err);
		responderrmsg(op, Eservererror, nil);
		donech <-= 0;
		return;
	}

	# to make sure our caller can return (e.g. when writing to remote fails)
	{
		_cgiproc(path, op, cgipath, cgiaction, cgitype, length, needcontinue, timeopid, cgich);
	} exception {
	* =>	killch <-= timeopid;	# may already have been killed
	}
	donech <-= 0;
}

# note: this function must always kill the timeout proc on "normal" returns (i.e. without raising an exception).
_cgiproc(path: string, op: ref Op, cgipath, cgiaction: string, cgitype: int, length: big, needcontinue: int, timeopid: int, cgich: chan of int)
{
	# pid ends up in timeout(), this proc is killed if it doesn't respond timely
	pid := sys->pctl(0, nil);
	cgich <-= pid;

	id := op.id;
	req := op.req;
	resp := op.resp;

	if(debugflag) say(id, sprint("handling cgi request, cgipath %q cgiaction %q cgitype %s, pid %d timeopid %d", cgipath, cgiaction, cgitypes[cgitype], pid, timeopid));

	fd0, fd1: ref Sys->FD;
	if(cgitype == Scgi) {
		scgidialch <-= (cgiaction, replychan := chan of (ref Sys->FD, string));
		(sfd, serr) := <-replychan;
		if(serr != nil) {
			warn(op.id, serr);
			killch <-= timeopid;
			return responderrmsg(op, Eservererror, "Internal Server Error: Handler not reachable");
		}

		sreq := scgirequest(path, cgipath, req, op, length);
		if(sys->write(sfd, sreq, len sreq) != len sreq) {
			warn(id, sprint("write scgi request: %r"));
			killch <-= timeopid;
			return responderrmsg(op, Eservererror, nil);
		}
		fd0 = fd1 = sfd;
	} else {
		err: string;
		cgispawnch <-= (cgiaction, path, cgipath, op, length, replych := chan of (ref Sys->FD, ref Sys->FD, string));
		(fd0, fd1, err) = <-replych;
		if(err != nil) {
			warn(id, "cgispawn: "+err);
			killch <-= timeopid;
			return responderrmsg(op, Eservererror, nil);
		}
	}

	sb := bufio->fopen(fd1, Bufio->OREAD);
	if(sb == nil) {
		warn(id, sprint("bufio fopen cgi fd: %r"));
		killch <-= timeopid;
		return responderrmsg(op, Eservererror, nil);
	}

	if(length > big 0)
		spawn cgifunnel(op.id, op.inb, fd0, length);

	# we always want a "status: ..." line from the cgi program.  it would be better if we would
	# generate a "200 ok" if the status is missing, but we cannot parse the full http request
	# after we've already read the first line (with a header in it) from the iobuf...

	rerr: string;
	hdrs: ref Hdrs;
	for(;;) {
		l := sb.gets('\n');
		killch <-= timeopid;
		if(l == nil) {
			warn(id, "eof from cgi handler while reading response line");
			return responderrmsg(op, Eservererror, "Internal Server Error: EOF from handler");
		}
		l = l[:len l-1];
		if(l != nil && l[len l-1] == '\r')
			l = l[:len l-1];

		if(!str->prefix("status:", str->tolower(l))) {
			warn(id, sprint("bad cgi response line: %q", l));
			return responderrmsg(op, Eservererror, "Internal Server Error: Handler sent bad response line");
		}
		l = str->drop(l[len "status:":], " \t");
		(resp.st, resp.stmsg) = str->splitstrl(l, " ");
		if(resp.stmsg != nil)
			resp.stmsg = droptl(resp.stmsg[1:], " \t");
		if(len resp.st != 3 || str->drop(resp.st, "0-9") != "") {
			warn(id, sprint("bad cgi response line: %q", l));
			return responderrmsg(op, Eservererror, "Internal Server Error: Handler sent bad response line");
		}

		(hdrs, rerr) = Hdrs.read(sb);
		if(rerr != nil) {
			warn(id, "reading cgi headers: "+rerr);
			return responderrmsg(op, Eservererror, "Internal Server Error: Error reading headers from handler");
		}

		if(resp.st == "100") {
			needcontinue = 0;
			fprint(op.fd, "HTTP/1.1 100 Continue\r\n\r\n");
			continue;
		}
		if(needcontinue)
			fprint(op.fd, "HTTP/1.1 100 Continue\r\n\r\n");
		break;
	}

	elength := big -1;
	if(hdrs.has("content-length", nil)) {
		elengthstr := hdrs.get("content-length");
		if(elengthstr == nil || str->drop(elengthstr, "0-9") != "") {
			warn(id, sprint("bad cgi content-length header: %q", elengthstr));
			return responderrmsg(op, Eservererror, "Internal Server Error: Invalid content-length from handler");
		}
		op.length = elength = big elengthstr;
	}
	for(hl := hdrs.all(); hl != nil; hl = tl hl)
		resp.h.add((hd hl).t0, (hd hl).t1);

	accesslog(op);

	op.chunked = elength == big -1 && resp.version() >= HTTP_11;
	rerr = hresp(resp, op.fd, op.keepalive, op.chunked);
	if(rerr != nil) {
		warn(id, "writing response: "+rerr);
		return;
	}

	if(req.method == HEAD)
		return;

	for(;;) {
		n := sb.read(d := array[Sys->ATOMICIO] of byte, len d);
		if(n < 0)
			die(id, sprint("reading file: %r"));
		if(n == 0) {
			if(elength > big 0)
				die(id, "bad cgi body, message shorter than content-length specified");
			break;
		}
		if(elength > big 0) {
			if(big n > elength)
				die(id, "bad cgi body, message longer than content-length specified");
			elength -= big n;
		}
		# for last bit of data, verify that the handler is finished writing
		if(elength > big n || (big n == elength && sb.read(tmp := array[1] of byte, len tmp) != 0))
			die(id, "bad cgi body, message longer than content-length specified");
		hwrite(op, d[:n]);
	}
	hwriteeof(op);
	if(debugflag) say(id, "request done");
}

cgifunnel(id: int, b: ref Iobuf, sfd: ref Sys->FD, length: big)
{
	while(length > big 0) {
		need := Sys->ATOMICIO;
		if(big need > length)
			need = int length;
		n := b.read(d := array[need] of byte, len d);
		if(n < 0)
			die(id, sprint("cgi read: %r"));
		if(n == 0)
			die(id, "cgi read: premature eof");
		{
			if(sys->write(sfd, d, n) != n)
				die(id, sprint("cgi write: %r"));
		} exception {
		* =>	die(id, sprint("cgi write: write on broken pipe"));
		}
		length -= big n;
	}
}

cgivars(path, cgipath: string, op: ref Op, length: big, environ: list of (string, string)): list of (string, string)
{
	servername := op.req.h.get("host");
	if(servername == nil)
		servername = op.lhost;
	pathinfo := path[len cgipath:];
	query := op.req.url.query;
	if(query != nil)
		query = query[1:];
	return	("CONTENT_LENGTH",	string length)::
		("GATEWAY_INTERFACE",	"CGI/1.1")::
		("SERVER_PROTOCOL",	http->versionstr(op.req.version()))::
		("SERVER_NAME",		servername)::
		("REQUEST_METHOD",	op.req.methodstr)::
		("REQUEST_URI",		op.req.url.packpath())::
		("SCRIPT_NAME",		cgipath)::
		("PATH_INFO",		pathinfo)::
		("PATH_TRANSLATED",	pathinfo)::
		("QUERY_STRING",	query)::
		("SERVER_ADDR",		op.lhost)::
		("SERVER_PORT",		op.lport)::
		("REMOTE_ADDR",		op.rhost)::
		("REMOTE_PORT",		op.rport)::
		("SERVER_SOFTWARE",	Version)::
		environ;
}

scgirequest(path, scgipath: string, req: ref Req, op: ref Op, length: big): array of byte
{
	l := ("SCGI", "1")::cgivars(path, scgipath, op, length, environment);
	s := "";
	for(h := l; h != nil; h = tl h)
		s += (hd h).t0+"\0"+(hd h).t1+"\0";
	for(h = req.h.all(); h != nil; h = tl h)
		s += cgivar((hd h).t0)+"\0"+(hd h).t1+"\0";
	return netstring(s);
}

cgivar(s: string): string
{
	r := "HTTP_";
	for(i := 0; i < len s; i++)
		if(s[i] != '-')
			r[len r] = s[i];
		else
			r[len r] = '_';
	return str->toupper(r);
}

netstring(s: string): array of byte
{
	return array of byte (sprint("%d:", len s)+s+",");
}


hresp(resp: ref Resp, fd: ref Sys->FD, keepalive, chunked: int): string
{
	if(keepalive)
		resp.h.add("connection", "keep-alive");
	else
		resp.h.add("connection", "close");
	if(chunked) {
		resp.h.add("transfer-encoding", "chunked");
		resp.h.del("content-length", nil);
	}
	return resp.write(fd);
}

hwrite(op: ref Op, d: array of byte)
{
	if(len d == 0)
		return;

	if(op.chunked) {
		length := array of byte sprint("%x\r\n", len d);
		nd := array[len length+len d+2] of byte;
		nd[:] = length;
		nd[len length:] = d;
		nd[len length+len d:] = array of byte "\r\n";
		d = nd;
	}
	if(sys->write(op.fd, d, len d) != len d)
		die(op.id, sprint("writing response data: %r"));
}

hwriteeof(op: ref Op)
{
	if(op.chunked)
		fprint(op.fd, "0\r\n\r\n");
}

respond(op: ref Op, st: int, errmsgstr: string, ct: string)
{
	resp := op.resp;
	resp.st = string st;
	resp.stmsg = statusmsg(st);
	if(ct != nil)
		resp.h.set("content-type", ct);

	op.chunked = 0;
	errmsg := array of byte errmsgstr;
	op.length = big len errmsg;
	if(!(st >= 100 && st < 200 || st == 204 || st == 304))
		resp.h.set("content-length", string op.length);

	err := hresp(resp, op.fd, op.keepalive, op.chunked);
	if(err != nil)
		die(op.id, "writing error response: "+err);

	if(errmsgstr != nil && (op.req == nil || op.req.method != HEAD)) {
		hwrite(op, errmsg);
		hwriteeof(op);
	}

	accesslog(op);
}

responderr(op: ref Op, st: int)
{
	return respond(op, st, nil, nil);
}

responderrmsg(op: ref Op, st: int, errmsg: string)
{
	if(errmsg == nil)
		errmsg = statusmsg(st);
	return respond(op, st, mkhtml(sprint("%d - %s", st, errmsg)), "text/html; charset=utf-8");
}

statusmsg(code: int): string
{
	for(i := 0; i < len statusmsgs && statusmsgs[i].t0 <= code; i++)
		if(code == statusmsgs[i].t0)
			return statusmsgs[i].t1;
	raise sprint("missing status message for code %d", code);
}

dnslookup(dnsch: chan of string, respch: chan of (string, string))
{
	path := "/net/dns";
	for(;;) {
		ip := <-dnsch;

		revip, err: string;
		l: list of string;
		fd := sys->open(path, Sys->ORDWR);
		if(fd == nil) {
			err = sprint("open %q: %r", path);
		} else {
			(l, err) = reversedns(fd, ip);
			if(len l > 1)
				err = "too many results";
			else if(len l == 0 && err == nil)
				err = "no results";
			else if(len l == 1)
				revip = hd l;
			
		}
		respch <-= (revip, err);
	}
}

reversedns(fd: ref Sys->FD, ip: string): (list of string, string)
{
	(nil, l) := sys->tokenize(ip, ".");
	if(len l != 4)
		return (nil, "invalid ip address");
	dst := "";
	for(; l != nil; l = tl l)
		dst = hd l+"."+dst;
	dst += "in-addr.arpa";
	if(fprint(fd, "%s ptr", dst) < 0)
		return (nil, sprint("%r"));
	r: list of string;
	for(;;) {
		have := sys->read(fd, d := array[1024] of byte, len d);
		if(have < 0)
			return (nil, sprint("read: %r"));
		if(have == 0)
			break;
		(nil, l) = sys->tokenize(string d[:have], " \t");
		if(len l != 3 || hd tl l != "ptr" || hd l != dst)
			continue;
		r = (hd tl tl l)::r;
	}
	return (r, nil);
}

logger()
{
	accessfd: ref Sys->FD;
	backlog := array[0] of (string, string);

	ip, msg: string;
	respch := chan of (string, string);
	dnsch := chan of string;

	spawn dnslookup(dnsch, respch);

	for(;;) alt {
	(logip, s) := <-logch =>
		if(accessfd == nil)
			continue;
		if(msg != nil) {
			na := array[len backlog+1] of (string, string);
			na[:] = backlog;
			na[len backlog] = (logip, s);
			backlog = na;
		} else {
			msg = s;
			ip = logip;
			dnsch <-= logip;
		}

	(revip, err) := <-respch =>
		if(err != nil)
			say(0, sprint("reverse dns lookup %q: %s", ip, err));
		if(fprint(accessfd, "%s %q\n", msg, revip) < 0)
			warn(0, sprint("writing access log message: %r"));
		msg = nil;
		if(len backlog != 0) {
			(ip, msg) = backlog[0];
			backlog = backlog[1:];
			dnsch <-= ip;
		}

	accessfd = <-logfdch =>
		;
	}
}

accesslog(op: ref Op)
{
	if(op.cfgs.accessfd == nil && !debugflag)
		return;

	length := "";
	if(!op.chunked && op.length >= big 0)
		length = string op.length;

	s := sprint("%d %d %s!%s %s!%s %q %q %q HTTP/%d.%d %q %q %q %q %q",
		op.id, op.now,
		op.rhost, op.rport, op.lhost, op.lport,
		op.req.methodstr, op.req.h.get("host"), op.req.url.path+op.req.url.query,
		op.req.major, op.req.minor, op.resp.st, op.resp.stmsg,
		length,
		op.req.h.get("user-agent"), op.req.h.get("referer"));
	logch <-= (op.rhost, s);
	if(debugflag) say(op.id, "accesslog: "+s);
}

splithost(s: string): (string, string)
{
	host: string;
	if(str->prefix("[", s)) {
		# "[ip6]"
		(ip, rem) := str->splitstrl(s[1:], "]");
		if(!str->prefix("]", rem))
			return (nil, nil);
		s = rem[1:];
		(ok, ipaddr) := IPaddr.parse(ip);
		if(ok != 0)
			return (nil, nil);
		# canonical form, eg for ipv4 mapped on ipv6
		host = ipaddr.text();
	} else
		(host, s) = str->splitstrl(s, ":");
	if(str->prefix(":", s))
		s = s[1:];
	return (host, s);
}

suffix(suf, s: string): int
{
	if(len suf > len s)
		return 0;
	return suf == s[len s-len suf:];
}

mimetype(cfgs: ref Cfgs, path: string): string
{
	for(t := cfgs.usertypes; t != nil; t = tl t)
		if(suffix((hd t).t0, path))
			return (hd t).t1;
	for(i := 0; i < len mimetypes; i++)
		if(suffix(mimetypes[i].t0, path))
			return mimetypes[i].t1;
	if(!haschar(str->splitstrr(path, "/").t1, '.'))
		return "text/plain; charset=utf-8";	# for mkfile, README, etc.
	return nil;
}

hasbody(req: ref Req): int
{
	return req.h.has("content-length", nil) || req.h.has("transfer-encoding", nil);
}

days := array[] of {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"};
months := array[] of {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};

httpdate(t: int): string
{
	tm := daytime->gmt(t);
	return sprint("%s, %02d %s %d %02d:%02d:%02d GMT", days[tm.wday], tm.mday, months[tm.mon], tm.year+1900, tm.hour, tm.min, tm.sec);
}

readtoken(s: string): (string, string, string)
{
	for(i := 0; i < len s; i++)
		if(s[i] < ' ')
			return (nil, nil, "invalid control characters found");
		else if(str->in(s[i], "()<>@,;:\\\"/[]?={} \t"))
			break;
	return (s[:i], s[i:], nil);
}

# read list of bare tokens
tokenize(s: string): (list of string, string)
{
	token, err: string;
	l: list of string;
	for(;;) {
		(token, s, err) = readtoken(s);
		if(err != nil)
			return (nil, err);
		if(token != nil)
			l = token::l;
		s = str->drop(s, " \t");
		if(s == nil)
			break;
		if(s[0] != ',')
			return (nil, "expected comma as separator");
		s = str->drop(s, ", \t");
	}
	return (rev(l), nil);
}

# read double quoted token in header.
# for http/1.1 a backslash may be used for escaping, not for http/1.0
readqs(s: string, v: int): (string, string, string)
{
	if(s == nil)
		return (nil, nil, nil);
	if(s[0] != '"')
		return (nil, s, nil);
	r := "\"";
	for(i := 1; i < len s; i++)
		if(s[i] < ' ')
			return (nil, nil, "invalid control character found inside quoted string");
		else if(v >= HTTP_11 && s[i] == '\\' && i+1 < len s && s[i+1] == '"')
			r[len r] = s[++i];
		else {
			r[len r] = s[i];
			if(s[i] == '"')
				return (r, s[i+1:], nil);
		}
	return (nil, nil, "quoted string not ended");
}

# read list of double-quoted strings
tokenizeqs(s: string, v: int): (list of string, string)
{
	r: list of string;
	qs, err: string;
	for(;;) {
		(qs, s, err) = readqs(s, v);
		if(err != nil)
			return (nil, err);
		if(qs != nil)
			r = qs::r;
		s = str->drop(s, " \t");
		if(s == nil)
			break;
		if(s[0] != ',')
			return (nil, "expected comma as separator");
		s = str->drop(s, ", \t");
	}
	return (rev(r), nil);
}

# i'm not going to parse three different date formats where a simple unix epoch integer would have sufficed.
# death to the bloat monster!
parsehttpdate(s: string): int
{
	mday, mon, year, hour, min, sec: int;

	# sys-tokenize allows too much whitespace, but well...
	(n, tokens) := sys->tokenize(s, " ");
	if(n != 6 || len hd tokens != 4 || (hd tokens)[3] != ',' || index(days, (hd tokens)[:3]) < 0)
		return 0;
	if((mon = index(months, hd tl tl tokens)) < 0)
		return 0;
	(hn, htokens) := sys->tokenize(hd tl tl tl tl tokens, ":");
	if(hn != 3)
		return 0;
	mday = int hd tl tokens;
	year = int hd tl tl tl tokens;
	hour = int hd htokens;
	min = int hd tl htokens;
	sec = int hd tl tl htokens;

	# last arg should be seconds offset for timezone, "luckily" http allows only gmt...
	return daytime->tm2epoch(ref Daytime->Tm(sec, min, hour, mday, mon, year-1900, 0, 0, s[1:], 0));
}

parserange(version: int, rangehdr: (int, string), length: big): (int, list of ref (big, big))
{
	(haverange, range) := rangehdr;
	if(!haverange || !(version >= HTTP_11))
		return (1, nil);

	if(!str->prefix("bytes", range))
		return (0, nil);
	range = range[len "bytes":];
	range = str->drop(range, " \t");
	if(!str->prefix("=", range))
		return (0, nil);
	range = str->drop(range[1:], " \t");

	# warning: range header parsing is fairly abstruse
	r: list of ref (big, big);
	valid := 0;
	for(l := sys->tokenize(range, ",").t1; l != nil; l = tl l) {
		s := strip(hd l, " \t");
		if(s == nil)
			continue;
		if(s[0] == '-') {
			# single (negative) byte offset relative to end of file
			s = str->drop(s[1:], " \t");
			if(s == nil || str->drop(s, "0-9") != nil)
				return (1, nil);
			if(big s != big 0)
				valid = 1;
			i := length - big s;
			if(i < big 0)
				i = big 0;
			if(i >= length)
				i = length - big 1;
			r = ref (i, length)::r;
		} else {
			(first, last) := str->splitstrl(s, "-");
			if(stripws(str->drop(first, "0-9")) != nil || last == nil || str->drop(stripws(last[1:]), "0-9") != nil)
				return (1, nil);
			f := big first;
			e := length;
			last = stripws(last[1:]);
			if(last != nil)
				e = big last+big 1;
			if(e > length)
				e = length;
			if(f > e)
				return (1, nil);
			if(f < length)
				valid = 1;
			r = ref (f, e)::r;
		}
	}
	return (valid, rev(r));
}

listingssort(cfg: ref Cfg)
{
	sort(cfg.listings, listingge);
}

sort[T](a: array of T, ge: ref fn(a, b: T): int)
{
	for(i := 1; i < len a; i++) {
		tmp := a[i];
		for(j := i; j > 0 && ge(a[j-1], tmp); j--)
			a[j] = a[j-1];
		a[j] = tmp;
	}
}

listingge(e1, e2: ref (string, int)): int
{
	return len e1.t0 < len e2.t0;
}

strip(s, cl: string): string
{
	return droptl(str->drop(s, cl), cl);
}

stripws(s: string): string
{
	return strip(s, " \t");
}

droptl(s, cl: string): string
{
	while(s != nil && str->in(s[len s-1], cl))
		s = s[:len s-1];
	return s;
}

index(a: array of string, s: string): int
{
	for(i := 0; i < len a; i++)
		if(a[i] == s)
			return i;
	return -1;
}

readaddr(id: int, path: string): (string, string)
{
	(s, err) := readfileline(path, 256);
	if(err != nil)
		die(id, err);
	(lhost, lport) := str->splitstrl(s, "!");
	if(lport != nil)
		lport = lport[1:];
	return (lhost, lport);
}

readfileline(path: string, maxsize: int): (string, string)
{
	fd := sys->open(path, Sys->OREAD);
	if(fd == nil)
		return (nil, sprint("open %s: %r", path));
	n := sys->read(fd, buf := array[maxsize] of byte, len buf);
	if(n < 0)
		return (nil, sprint("read %s: %r", path));
	s := string buf[:n];
	if(s != nil && s[len s-1] == '\n')
		s = s[:len s-1];
	return (s, nil);
}

readtime(): int
{
	n := sys->pread(timefd, d := array[64] of byte, len d, big 0);
	if(n < 0)
		fail(sprint("reading time: %r"));
	return int ((big string d[:n])/big 1000000);
}

byte2str(a: array of byte): string
{
	s := "";
	for(i := 0; i < len a; i++)
		s += sprint("%02x", int a[i]);
	return s;
}

sha1(a: array of byte): string
{
	r := array[keyring->SHA1dlen] of byte;
	keyring->sha1(a, len a, r, nil);
	return byte2str(r);
}

listhas(l: list of string, s: string): int
{
	for(; l != nil; l = tl l)
		if(str->tolower(hd l) == s)
			return 1;
	return 0;
}

haschar(s: string, c: int): int
{
	for(i := 0; i < len s; i++)
		if(s[i] == c)
			return 1;
	return 0;
}

hasmethod(l: list of string, v: string): int
{
	for(; l != nil; l = tl l)
		if(hd l == v)
			return 1;
	return 0;
}

substr(sub, s: string): int
{
	return str->prefix(sub, str->splitstrl(s, sub).t1);
}

rev[T](l: list of T): list of T
{
	r: list of T;
	for(; l != nil; l = tl l)
		r = hd l::r;
	return r;
}

kill(pid: int)
{
	fd := sys->open(sprint("/prog/%d/ctl", pid), Sys->OWRITE);
	if(fd != nil)
		fprint(fd, "kill");
}

say(id:int, s: string)
{
	msec := sys->millisec();
	if(debugflag)
		fprint(fildes(2), "%d %d, %s\n", msec, id, s);
	if(debugflag && errorfd != nil)
		fprint(errorfd, "%d %d, %s\n", msec, id, s);
}

warn(id: int, s: string)
{
	msec := sys->millisec();
	if(debugflag)
		fprint(fildes(2), "%d %d, %s\n", msec, id, s);
	if(errorfd != nil)
		fprint(errorfd, "%d %d, %s\n", msec, id, s);
}

die(id: int, s: string)
{
	warn(id, s);
	raise "fail:"+s;
}

fail(s: string)
{
	fprint(fildes(2), "%s\n", s);
	raise "fail:"+s;
}


# reading the config file

Cfgs.init(file: string): (ref Cfgs, string)
{
	db := Db.open(file);
	if(db == nil)
		return (nil, sprint("open config %q: %r", file));
	return Cfgs.read(file, db);
}

Cfgs.read(file: string, db: ref Db): (ref Cfgs, string)
{
	c := ref Cfgs(file, db, Cfg.new(), nil, nil, 0, 0, nil, nil);
	logfile: string;

	e: ref Dbentry; 
	(e, nil) = c.db.find(nil, "vhost");
	if(e != nil)
		c.vhostflag = 1;
	(e, nil) = c.db.find(nil, "novhost");
	if(e != nil)
		c.vhostflag = 0;

	(e, nil) = c.db.find(nil, "debug");
	if(e != nil)
		c.debugflag = 1;
	(e, nil) = c.db.find(nil, "nodebug");
	if(e != nil)
		c.debugflag = 0;

	(e, nil) = c.db.find(nil, "ctlchan");
	if(e != nil) {
		s := e.findfirst("ctlchan");
		if(s != nil)
			ctlchan = s;
	}

	(e, nil) = c.db.find(nil, "accesslog");
	if(e != nil) {
		s := e.findfirst("accesslog");
		if(s != nil)
			logfile = s;
	}

	ptr: ref Attrdb->Dbptr;
	attr := "mime";
	for(;;) {
		(e, ptr) = c.db.find(ptr, attr);
		if(e == nil)
			break;
		ext := e.findfirst("ext");
		mtype := e.findfirst("type");
		if(ext == nil || mtype == nil)
			return (nil, sprint("bad mime type, ext=%q type=%q", ext, mtype));
		c.usertypes = ref (ext, mtype)::c.usertypes;
	}
	ptr = nil;

	attr = "announce";
	for(;;) {
		(e, ptr) = c.db.find(ptr, attr);
		if(e == nil)
			break;
		c.addrs = e.findfirst("announce")::c.addrs;
	}
	ptr = nil;

	attr = "host";
	for(;;) {
		(e, ptr) = c.db.find(ptr, attr);
		if(e == nil)
			break;
		host := e.findfirst("host");
		port := e.findfirst("port");
		if(port == nil)
			port = "80";
		(cfg, err) := Cfg.read(e, port);
		if(err != nil)
			return (nil, err);
		if(host == "*")
			host = "";
		cfg.host = host;
		cfg.port = string int port;
		if(host == nil)
			c.default = cfg;
		c.cfgs = (host, port, cfg)::c.cfgs;
	}
	ptr = nil;

	attr = "alias";
	for(;;) {
		(e, ptr) = c.db.find(ptr, attr);
		if(e == nil)
			break;
		host := e.findfirst("host");
		port := e.findfirst("port");
		usehost := e.findfirst("usehost");
		useport := e.findfirst("useport");
		if(port == nil)
			port = "80";
		if(usehost == nil)
			usehost = host;
		if(useport == nil)
			useport = port;
		if(usehost == host && useport == port)
			return (nil, "alias line aliases host and port to itself, ignoring");
		cfg := c.find(usehost, string int useport);
		if(cfg == nil)
			return (nil, sprint("alias references non-existing usehost=%q useport=%q", usehost, useport));
		c.cfgs = (host, string int port, cfg)::c.cfgs;
	}
	ptr = nil;

	if(logfile != nil) {
		c.accessfd = sys->open(logfile, Sys->OWRITE);
		if(c.accessfd == nil)
			return (nil, sprint("open logfile %q: %r", logfile));
		sys->seek(c.accessfd, big 0, Sys->SEEKEND);
	}
	c.usertypes = rev(c.usertypes);

	return (c, nil);
}

Cfgs.find(c: self ref Cfgs, host, port: string): ref Cfg
{
	for(l := c.cfgs; l != nil; l = tl l) {
		(chost, cport, config) := hd l;
		if(host == chost && port == cport)
			return config;
	}
	return c.default;
}


Cfg.new(): ref Cfg
{
	return ref Cfg("", "80", array[0] of ref (string, int), -1, nil, nil, nil, nil, nil, nil);
}

Cfg.read(e: ref Dbentry, defaultport: string): (ref Cfg, string)
{
	cfg := Cfg.new();

	for(l := list of {"cachesecs"}; l != nil; l = tl l) {
		for(r := e.find(hd l); r != nil; r = tl r) {
			for(attrs := (hd r).t1; attrs != nil; attrs = tl attrs) {
				val := (hd attrs).val;
				case (hd attrs).attr {
				"cachesecs" =>
					cfg.cachesecs = int val;
				* =>
					warn(0, sprint("ignoring config attribute %q", (hd attrs).attr));
				}
			}
		}
	}

	for(l = list of {"listings", "nolistings", "listen", "redir", "auth", "index", "cgi", "scgi", "gzip"}; l != nil; l = tl l) {
		attr := hd l;
		for(r := e.find(attr); r != nil; r = tl r) {
			(tups, nil) := hd r;
			case attr {
			"listings" or "nolistings" =>
				what := attr == "listings";
				for(pl := tups.find("path"); pl != nil; pl = tl pl) {
					nl := array[len cfg.listings+1] of ref (string, int);
					nl[:] = cfg.listings;
					nl[len cfg.listings] = ref ((hd pl).val, what);
					cfg.listings = nl;
				}

			"listen" =>
				ip := tups.find("ip");
				port := tups.find("port");
				if(ip == nil)
					return (nil, sprint("missing ip in listen line"));
				ipstr := (hd ip).val;
				(ok, ipaddr) := IPaddr.parse(ipstr);
				if(ok != 0)
					return (nil, sprint("invalid ip address: %q", ipstr));
				portstr := defaultport;
				if(port != nil)
					portstr = (hd port).val;
				cfg.listens = (ref (ipaddr.text(), string int portstr))::cfg.listens;

			"redir" =>
				src := tups.find("src");
				dst := tups.find("dst");
				if(src == nil || dst == nil)
					return (nil, "missing src or dst in redir line");
				(repl, rerr) := Repl.parse((hd src).val, (hd dst).val);
				if(rerr != nil)
					return (nil, "parsing redir: "+rerr);
				cfg.redirs = repl::cfg.redirs;

			"auth" =>
				path := tups.find("path");
				realm := tups.find("realm");
				user := tups.find("user");
				pass := tups.find("pass");
				if(path == nil || realm == nil || user == nil || pass == nil)
					return (nil, "missing field in auth line, need path, realm, user and pass");
				if(haschar((hd realm).val, '"'))
					return (nil, "realm must not have double quote, not supported by http/1.0");
				cfg.auths = ref ((hd path).val, (hd realm).val, base64->enc(array of byte ((hd user).val+":"+(hd pass).val)))::cfg.auths;

			"index" =>
				for(file := tups.find("file"); file != nil; file = tl file)
					cfg.indexfiles = (hd file).val::cfg.indexfiles;

			"cgi" =>
				path := tups.find("path");
				cmd := tups.find("cmd");
				if(path == nil || cmd == nil)
					return (nil, "missing path or cmd in cgi line");

				methods: list of string;
				methodtups := tups.find("methods");
				if(methodtups != nil)
					methods = sys->tokenize((hd methodtups).val, " ,").t1;
				cfg.cgipaths = ref ((hd path).val, (hd cmd).val, methods, Cgi)::cfg.cgipaths;

			"scgi" =>
				path := tups.find("path");
				addr := tups.find("addr");
				if(path == nil || addr == nil)
					return (nil, "missing path or addr in scgi line");

				methods: list of string;
				methodtups := tups.find("methods");
				if(methodtups != nil)
					methods = sys->tokenize((hd methodtups).val, " ,").t1;
				cfg.cgipaths = ref ((hd path).val, (hd addr).val, methods, Scgi)::cfg.cgipaths;
			"gzip" =>
				for(pl := tups.find("path"); pl != nil; pl = tl pl)
					cfg.gzpaths = (hd pl).val::cfg.gzpaths;
			}
		}
	}
	cfg.rev();
	return (cfg, nil);
}

Cfg.rev(cfg: self ref Cfg)
{
	cfg.listens = rev(cfg.listens);
	cfg.cgipaths = rev(cfg.cgipaths);
	cfg.indexfiles = rev(cfg.indexfiles);
	cfg.redirs = rev(cfg.redirs);
	cfg.auths = rev(cfg.auths);
	listingssort(cfg);
}


Repl.parse(restr, rulestr: string): (ref Repl, string)
{
	(re, err) := regex->compile(restr, 1);
	if(err != nil)
		return (nil, "bad regex: "+err);

	rule: list of ref (string, int);
	maxrepl := 0;
	for(;;) {
		(l, r) := str->splitstrl(rulestr, "$");
		if(r == nil) {
			rule = ref (l, -1)::rule;
			break;
		}
		r = r[1:];
		if(r != nil && r[0] == '$') {
			rule = ref (l+"$", -1)::rule;
			r = r[1:];
		} else {
			num := str->take(r, "0-9");
			if(num == nil)
				return (nil, "bad rule: $ not followed by number or dollar");
			n := int num;
			if(n > maxrepl)
				maxrepl = n;
			rule = ref (l, n)::rule;
			r = r[len num:];
		}
		rulestr = r;
	}
	rule = rev(rule);
	return (ref Repl(re, rule, maxrepl), nil);
}

Repl.apply(r: self ref Repl, s: string): (int, string, string)
{
	m := regex->executese(r.re, s, (0, len s), 1, 1);
	if(m == nil)
		return (0, nil, nil);
	if(r.maxrepl > len m-1)
		return (0, nil, "replacement group not in groups in regular expression");
	res := "";
	for(rl := r.rule; rl != nil; rl = tl rl) {
		(part, index) := *(hd rl);
		res += part;
		if(index == -1)
			continue;
		(b, e) := m[index];
		if(b == -1 || e == -1)
			return (0, nil, "replacement group did not match in regular expression");
		res += s[b:e];
	}
	return (1, res, nil);
}

%{
include "sys.m";
	sys: Sys;
	print, sprint, fprint, fildes: import sys;
include "draw.m";
include "arg.m";
include "bufio.m";
	bufio: Bufio;
	Iobuf: import bufio;
include "daytime.m";
	daytime: Daytime;
include "dict.m";
	dict: Dictionary;
	Dict: import dict;
include "lists.m";
	lists: Lists;
include "string.m";
	str: String;

# line from the log file
Log: adt {
	d:	ref Dict;
	time:	int;

	parse:	fn(s: string): (ref Log, string);
	get:	fn(l: self ref Log, s: string): (string, string);
};


# what comes after keyword "match"
Expr: adt {
	pick {
	And or Or or Eq or Ne =>
		p0, p1:	cyclic ref Expr;
	Param or Str =>
		s:	string;
	}

	eval:	fn(e: self ref Expr, l: ref Log): int;
	text:	fn(e: self ref Expr): string;
};

# keywords "limit", "sort" & "reverse"
Final: adt {
	pick {
	Limit =>
		n:	int;
	Sort or Normalise =>
		s:	string;
	Reverse =>
	Lump =>
		s, cutoff, str:	string;
	}

	text:	fn(f: self ref Final): string;
};

# a table, its requirements & progress/state
Table: adt {
	name, descr:	string;
	match:	ref Expr;
	per:	list of string;
	keep:	list of string;
	final:	list of ref Final;

	rows:	array of array of ref (string, int, int, int);  # string value, int value, extra values (used by avg)
	types:	array of int;

	colnames:	fn(t: self ref Table): list of string;
	apply:	fn(t: self ref Table, l: ref Log): string;
	finalize:	fn(t: self ref Table, f: ref Final);
	averages:	fn(t: self ref Table);
	write:	fn(t: self ref Table, fd: ref Sys->FD): string;
	writetabbed:	fn(t: self ref Table, fd: ref Sys->FD): string;
	text:	fn(t: self ref Table): string;
};

YYSTYPE: adt {
	tab, word, str:	string;
	table:	ref Table;
	match:	ref Expr;
};

YYLEX: adt {
	lval:	YYSTYPE;
	lex:	fn(l: self ref YYLEX): int;
	error:	fn(l: self ref YYLEX, msg: string);
};

%}

%module Logparse {
	init:	fn(nil: ref Draw->Context, args: list of string);
}


%type <match>	exprs expr param

%token <tab>	TAB
%token <str>	STR
%token <word>	WORD
%token DESCR PER KEEP MATCH SORT LIMIT REVERSE NORMALISE LUMP END

%token AND
%token OR
%token EQ NE

%%

top:
	top table { tabs = curtab::tabs; }
	| { tabs = lists->reverse(tabs); }
	;

table:
	TAB { curtab = ref zerotab; curtab.name = $1; } descr matches pers keeps finals
	END {
		curtab.per = lists->reverse(curtab.per);
		curtab.keep = lists->reverse(curtab.keep);
		curtab.final = lists->reverse(curtab.final);
	}
	;

descr:
	DESCR STR { curtab.descr = $2; }
	|
	;

matches:
	matches match
	|
	;

match:
	MATCH exprs {
		if(curtab.match == nil)
			curtab.match = $2;
		else
			curtab.match = ref Expr.And(curtab.match, $2);
	}
	;

pers:
	pers per
	|
	;

per:
	PER WORD { curtab.per = $2::curtab.per; }
	;

exprs:
	expr AND expr	{ $$ = ref Expr.And ($1, $3); }
	| expr OR expr	{ $$ = ref Expr.Or ($1, $3); }
	| expr
	;

expr:
	param EQ param	{ $$ = ref Expr.Eq($1, $3); }
	| param NE param	{ $$ = ref Expr.Ne($1, $3); }
	| param
	;

param:
	WORD	{ $$ = ref Expr.Param ($1); }
	| STR	{ $$ = ref Expr.Str ($1); }
	;

keeps:
	keeps keep
	|
	;

keep:
	KEEP WORD { 
		if(!isaggregate($2) && len curtab.per != 0 && !has($2, curtab.per))
			fail(sprint("%s:%d: cannot have non-aggregate 'keep' key %q that is not specified as 'per'", tabfile, lineno, $2));
		curtab.keep = $2::curtab.keep;
	}
	;

finals:
	finals final
	|
	;

final:
	SORT WORD	{
		if(fieldindex(curtab, $2) < 0)
			fail(sprint("%s:%d: sort key %#q must be in 'per' or 'keep' statement too", tabfile, lineno, $2));
		curtab.final = ref Final.Sort ($2)::curtab.final;
	}
	| LIMIT WORD	{ curtab.final = ref Final.Limit (int $2)::curtab.final; }
	| REVERSE	{ curtab.final = ref Final.Reverse::curtab.final; }
	| NORMALISE WORD	{
		if(fieldindex(curtab, $2) < 0)
			fail(sprint("%s:%d: normalise key %#q must be in 'per' or 'keep' statement too", tabfile, lineno, $2));
		curtab.final = ref Final.Normalise ($2)::curtab.final;
	}
	| LUMP WORD WORD STR {
		if(fieldindex(curtab, $2) < 0)
			fail(sprint("%s:%d: lump key %#q must be in 'per' or 'keep' statement too", tabfile, lineno, $2));
		if(str->toint($3, 10).t1 != nil)
			fail(sprint("%s:%d: bad threshold, must be numeric", tabfile, lineno));
		curtab.final = ref Final.Lump ($2, $3, $4)::curtab.final;
	}
	;

%%

dflag: int;
qflag: int;

btab: ref Iobuf;
done: int;
zerotab: Table;

curtab: ref Table;
tabs: list of ref Table;
lineno := 1;

Searchblocksize: con big (32*1024);

tabfile, logfile: string;

init(nil: ref Draw->Context, args: list of string)
{
	sys = load Sys Sys->PATH;
	bufio = load Bufio Bufio->PATH;
	daytime = load Daytime Daytime->PATH;
	dict = load Dictionary Dictionary->PATH;
	str = load String String->PATH;
	lists = load Lists Lists->PATH;
	arg := load Arg Arg->PATH;

	starttime := endtime := -1;

	arg->init(args);
	arg->setusage(arg->progname()+" [-dq] [-s starttime] [-e endtime] tabfile log");
	while((c := arg->opt()) != 0)
		case c {
		'd' =>	dflag++;
		's' =>	starttime = int arg->earg();
		'e' =>	endtime = int arg->earg();
		'q' =>	qflag++;
		* =>	arg->usage();
		}
	args = arg->argv();
	if(len args != 2)
		arg->usage();
	tabfile = hd args;
	logfile = hd tl args;

	btab = bufio->open(tabfile, Bufio->OREAD);
	if(btab == nil)
		fail(sprint("open %q: %r", tabfile));

	blog := bufio->open(logfile, Bufio->OREAD);
	if(blog == nil)
		fail(sprint("open %q: %r", logfile));

	done = 0;
	lex := ref YYLEX;
	yyparse(lex);

	tabs = lists->reverse(tabs);
	if(dflag) {
		for(t := tabs; t != nil; t = tl t)
			warn((hd t).text()+"\n");
	}

	# seek to starttime if there is one
	if(starttime >= 0) {
		start := big 0;
		end := blog.seek(start, Bufio->SEEKEND);

		while((diff := end-start) >= big 2*Searchblocksize) {
			mid := start + diff/big 2;
			blog.seek(mid, Bufio->SEEKSTART);
			blog.gets('\n');
			line := blog.gets('\n');
			if(line != nil)
				(l, err) := Log.parse(line);
			if(line == nil || err != nil)
				break;  # current "start" is just fine

			if(l.time >= starttime)
				end = mid;
			else
				start = mid;
		}

		blog.seek(start, Bufio->SEEKSTART);
		blog.gets('\n');
	}

	# for each line in the log file...
	for(;;) {
		line := blog.gets('\n');
		if(line == nil)
			break;
		(l, err) := Log.parse(line);
		if(err != nil)
			fail("parselog: "+err);

		if(starttime >= 0 && l.time < starttime)
			continue;
		if(endtime >= 0 && l.time > endtime)
			break;

		# try each table to see if they "accept" it (by match expressions)
		for(t := tabs; t != nil; t = tl t) {
			tab := hd t;
			if(tab.match == nil || tab.match.eval(l))
				if((err = tab.apply(l)) != nil)
					fail(err);
		}
	}

	# now we reverse/limit/sort/normalise/lump and then recalculate averages
	for(t := tabs; t != nil; t = tl t) {
		tab := hd t;
		for(f := tab.final; f != nil; f = tl f)
			tab.finalize(hd f);

		tab.averages();
	}

	fd := sys->fildes(1);
	for(t = tabs; t != nil; t = tl t) {
		tab := hd t;
		err := "";
		if(qflag)
			err = tab.write(fd);
		else
			err = tab.writetabbed(fd);
		if(err != nil) {
			warn(sprint("write %q: %s", tab.name, err));
			continue;
		}
		sys->fprint(fd, "\n");
	}
}


YYLEX.error(nil: self ref YYLEX, err: string)
{
	fail(sprint("%s:%d: %s", tabfile, lineno, err));
}

YYLEX.lex(lex: self ref YYLEX): int
{
	if(done)
		return -1;
	for(;;){
		c := btab.getc();
		case c {
		'"' =>
			# read until end of string, two consecutive ones escape
			s: string;
			for(;;) {
				case c = btab.getc() {
				Bufio->EOF or Bufio->ERROR =>
					return -1;
				'"' =>
					c = btab.getc();
					if(c != '"') {
						btab.ungetc();
						lex.lval.str = s;
						return STR;
					}
					s[len s] = '"';
				* =>
					s[len s] = c;
				}
			}

		' ' or '\t' =>
			;
		'\n' =>
			lineno++;
			n := 0;
			for(;;) {
				c = btab.getc();
				if(c != '\n')
					break;
				lineno++;
				n++;
			}
			btab.ungetc();
			if(n > 0)
				return END;

		'!' =>
			c = btab.getc();
			if(c == '=')
				return NE;
			btab.ungetc();
			return -1;

		'=' =>
			c = btab.getc();
			if(c == '=')
				return EQ;
			btab.ungetc();
			return -1;

		'a' to 'z' or '0' to '9' or '(' or ')' =>
			s: string;
			s[len s] = c;
			for(;;) {
				case c = btab.getc() {
				'a' to 'z' or '0' to '9' or '(' or ')' or ':' =>
					s[len s] = c;
				Bufio->ERROR =>
					return -1;
				* =>
					if(c != Bufio->EOF)
						btab.ungetc();
					if(s[len s-1] == ':') {
						lex.lval.tab = s[:len s-1];
						return TAB;
					}

					case s {
					"descr" =>	return DESCR;
					"per" =>	return PER;
					"keep" =>	return KEEP;
					"match" =>	return MATCH;
					"sort" =>	return SORT;
					"limit" =>	return LIMIT;
					"reverse" =>	return REVERSE;
					"normalise" =>	return NORMALISE;
					"lump" =>	return LUMP;
					"and" =>	return AND;
					"or" =>		return OR;
					* =>
						lex.lval.word = s;
						return WORD;
					}
				}
			}

		Bufio->ERROR =>
			return -1;
		Bufio->EOF =>
			done = 1;
			return END;
		}
	}
}


keys := array[] of {
"msec",
"time",
"fromaddr",
"toaddr",
"method",
"host",
"path",
"httpversion",
"status",
"statusmsg",
"size",
"useragent",
"referer",
"fromdns",
};

Log.parse(s: string): (ref Log, string)
{
	l := str->unquoted(s);
	if(len l != len keys)
		return (nil, sprint("wrong number of tokens, want %d, have %d", len keys, len l));
	v := l2a(l);
	d := ref Dict;
	for(i := 0; i < len keys; i++)
		d.add((keys[i], v[i]));
	time := int v[1];
	return (ref Log (d, time), nil);
}


substr(sub, s: string): int
{
	return str->splitstrl(s, sub).t1 != nil;
}

aggrs := array[] of {
"count", "sum(size)", "avg(size)",
};
isaggregate(s: string): int
{
	for(i := 0; i < len aggrs; i++)
		if(aggrs[i] == s)
			return 1;
	return 0;
}

isavg(s: string): int
{
	return s == "avg(size)";
}

Log.get(l: self ref Log, s: string): (string, string)
{
	val := l.d.lookup(s);
	if(val != nil)
		return (val, nil);

	# might have empty value.  annoying dict interface...
	for(k := l.d.keys(); k != nil; k = tl k)
		if(hd k == s)
			return ("", nil);

	case s {
	"useragentname(useragent)" =>
		ua := l.get("useragent").t0;
		return (str->splitstrl(ua, " ").t0, nil);

	"bothuman(useragent)" =>
		bots := array[] of {
		"bot",
		"yahoo! slurp",
		};
		v := str->tolower(l.get("useragent").t0);
		for(i := 0; i < len bots; i++)
			if(substr(bots[i], v))
				return ("bot", nil);
		return ("human", nil);

	"fromip" =>
		v := l.get("fromaddr").t0;
		return (str->splitstrl(v, "!").t0, nil);

	"subnet16(fromip)" =>
		v := l.get("fromaddr").t0;
		e := sys->tokenize(v, ".").t1;
		return (hd e+"."+hd tl e, nil);

	"dnstld(dns)" =>
		v := l.get("fromdns").t0;
		if(v == nil)
			return ("<no reverse dns>", nil);
		return (str->splitstrr(v, ".").t1, nil);

	"domain(referer)" =>
		v := l.get("referer").t0;
		if(v == nil)
			return ("<no referer>", nil);
		if(str->prefix("http://", v))
			v = v[len "http://":];
		else if(str->prefix("https://", v))
			v = v[len "https://":];
		else
			return ("<malformed>", nil);
		return (str->splitstrl(v, "/").t0, nil);

	"dayofweek(time)" =>
		days := array[] of {
		"sun", "mon", "tue", "wed", "thu", "fri", "sat",
		};
		tm := daytime->gmt(l.time);
		return (days[tm.wday], nil);

	"date(time)" =>
		tm := daytime->gmt(l.time);
		return (sprint("%04d-%02d-%02d", 1900+tm.year, tm.mon+1, tm.mday), nil);

	"hour(time)" =>
		tm := daytime->gmt(l.time);
		return (sprint("%02d", tm.hour), nil);

	"week(time)" =>
		tm := daytime->gmt(l.time);
		week := 1+tm.yday/7;
		firstsat := tm.yday%7 + (6-tm.wday);
		if(firstsat != 6 && tm.yday > firstsat)
			week++;
		return (string week, nil);

	"month(time)" =>
		months := array[] of {
		"jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"
		};
		tm := daytime->gmt(l.time);
		return (months[tm.mon], nil);

	"extension(path)" =>
		v := l.get("path").t0;
		rem: string;
		(rem, v) = str->splitr(v, "/.");
		if(rem != nil && rem[len rem-1] == '/')
			v = "";
		else
			v = "."+v;
		return (v, nil);

	"major(status)" =>
		v := l.get("status").t0;
		return (v[0:1], nil);

	* =>
		return (nil, "no such key/derivative: "+s);
	}
}


eval(e: ref Expr, l: ref Log): string
{
	pick ee := e {
	And =>
		if(eval(ee.p0, l) != "" && eval(ee.p1, l) != "")
			return "true";
		return "";
	Or =>
		if(eval(ee.p0, l) != "" || eval(ee.p1, l) != "")
			return "true";
		return "";
	Eq =>
		if(eval(ee.p0, l) == eval(ee.p1, l))
			return "true";
		return "";
	Ne =>
		if(eval(ee.p0, l) != eval(ee.p1, l))
			return "true";
		return "";
	Param =>
		(v, err) := l.get(ee.s);
		if(err != nil) {
			warn(sprint("eval %q: %s", ee.s, err));
			return "";
		}
		return v;
	Str =>
		return ee.s;
	}
}

Expr.eval(e: self ref Expr, l: ref Log): int
{
	return eval(e, l) != "";
}


Table.colnames(t: self ref Table): list of string
{
	v: list of string;
	for(l := t.per; l != nil; l = tl l)
		v = hd l::v;
	for(l = t.keep; l != nil; l = tl l)
		v = hd l::v;
	return lists->reverse(v);
}


Table.apply(t: self ref Table, l: ref Log): string
{
	# find row, or reuse existing row in case of aggregates
	(row, err) := getrow(t, l);

	i := len t.per;
	for(k := t.keep; err == nil && k != nil; k = tl k)
		err = setcol(row, i++, hd k, l, t.types);
	return err;
}

# NOTE: keep isaggregate() in sync!
setcol(row: array of ref (string, int, int, int), col: int, k: string, l: ref Log, types: array of int): string
{
	case k {
	"count" =>
		types[col] = 1;
		row[col].t1++;

	"sum(size)" =>
		types[col] = 1;
		v := l.get("size").t0;
		if(v != nil)
			row[col].t1 += int v;

	"avg(size)" =>
		types[col] = 1;
		v := l.get("size").t0;
		if(v != nil) {
			row[col].t2 += int v;
			row[col].t3++;
			row[col].t1 = row[col].t2/row[col].t3;
		}

	* =>
		(v, err) := l.get(k);
		if(err != nil)
			return err;
		row[col].t0 = v;
	}
	return nil;
}

getrow(t: ref Table, l: ref Log): (array of ref (string, int, int, int), string)
{
	# xxx should be done once during table init...
	if(t.types == nil)
		t.types = array[len t.per+len t.keep] of {* => 0};

	pvs := array[len t.per] of ref (string, int, int, int);
	i := 0;
	for(pers := t.per; pers != nil; pers = tl pers) {
		p := hd pers;
		(v, err) := l.get(p);
		if(err != nil)
			return (nil, err);
		pvs[i++] = ref (v, 0, 0, 0);
	}

	if(len pvs > 0) {
	nextrow:
		for(i = 0; i < len t.rows; i++) {
			for(j := 0; j < len pvs; j++)
				if(t.rows[i][j].t0 != pvs[j].t0)
					continue nextrow;
			return (t.rows[i], nil);
		}
	}

	ncol := len t.per+len t.keep;

	row := array[ncol] of {* => ref ("", 0, 0, 0)};
	row[:] = pvs;

	nrows := array[len t.rows+1] of array of ref (string, int, int, int);
	nrows[:] = t.rows;
	nrows[len t.rows] = row;
	t.rows = nrows;

	return (row, nil);
}

sort[T](a: array of T, ge: ref fn(a, b: T, field, numeric: int): int, field, numeric: int)
{
	for(i := 1; i < len a; i++) {
		tmp := a[i];
		for(j := i; j > 0 && ge(a[j-1], tmp, field, numeric); j--)
			a[j] = a[j-1];
		a[j] = tmp;
	}
}

# higher is first
rowge(a, b: array of ref (string, int, int, int), i, numeric: int): int
{
	if(numeric)
		return a[i].t1 < b[i].t1;
	return a[i].t0 < b[i].t0;
}

fieldindex(t: ref Table, s: string): int
{
	i := 0;
	for(l := t.per; l != nil; l = tl l)
		if(hd l == s)
			return i;
		else
			i++;
	for(l = t.keep; l != nil; l = tl l)
		if(hd l == s)
			return i;
		else
			i++;
	raise "unknown field requested: "+s;
}

Table.finalize(t: self ref Table, f: ref Final)
{
	if(t.rows == nil || len t.rows == 0)
		return;
	pick ff := f {
	Sort =>
		col := fieldindex(t, ff.s);
		sort(t.rows, rowge, col, t.types[col]);
	Limit =>
		if(len t.rows > ff.n)
			t.rows = t.rows[:ff.n];
	Reverse =>
		for(i := 0; i < len t.rows ; i++)
			(t.rows[i], t.rows[len t.rows-1-i]) = (t.rows[len t.rows-1-i], t.rows[i]);
	Normalise =>
		col := fieldindex(t, ff.s);
		if(t.types[col] != 1)
			fail(sprint("cannot normalise a non-integer"));
		total := 0;
		for(i := 0; i < len t.rows; i++)
			total += t.rows[i][col].t1;
		for(i = 0; i < len t.rows; i++)
			t.rows[i][col].t1 = 1000*t.rows[i][col].t1/total;
	Lump =>
		col := fieldindex(t, ff.s);
		if(t.types[col] != 1)
			fail(sprint("cannot lump a non-integer"));
		cutoff := int ff.cutoff;
		first := -1;
		i := 0;
		while(i < len t.rows) {
			r := t.rows[i];
			if(r[col].t1 <= cutoff) {
				if(first < 0) {
					first = i;
					for(j := 0; j < len t.types; j++)
						r[j].t0 = ff.str;
				} else {
					t.rows[first][col].t1 += r[col].t1;
					t.rows[first][col].t2 += r[col].t2;
					t.rows[first][col].t3 += r[col].t3;
					t.rows[i:] = t.rows[i+1:];
					t.rows = t.rows[:len t.rows-1];
					continue;
				}
			}
			i++;
		}
	}
}

Table.averages(t: self ref Table)
{
	i := 0;
	for(l := t.colnames(); l != nil; l = tl l) {
		if(isavg(hd l)) {
			for(j := 0; j < len t.rows; j++) {
				cell := t.rows[j][i];
				if(cell.t3 > 0)
					cell.t1 = cell.t2/cell.t3;
			}
		}
		i++;
	}
}

Table.write(t: self ref Table, fd: ref Sys->FD): string
{
	sys->fprint(fd, "%s: %s\n", t.name, t.descr);

	sys->fprint(fd, "%s\n", str->quoted(t.colnames()));
	for(i := 0; i < len t.rows; i++) {
		v: list of string;
		for(j := 0; j < len t.types; j++) {
			if(t.types[j])
				v = string t.rows[i][j].t1::v;
			else
				v = t.rows[i][j].t0::v;
		}
		sys->fprint(fd, "%s\n", str->quoted(lists->reverse(v)));
	}
	return nil;
}

Table.writetabbed(t: self ref Table, fd: ref Sys->FD): string
{
	sys->fprint(fd, "%s: %s\n", t.name, t.descr);
	for(l := t.per; l != nil; l = tl l)
		sys->fprint(fd, "\t%s", hd l);
	for(l = t.keep; l != nil; l = tl l)
		sys->fprint(fd, "\t%s", hd l);
	sys->fprint(fd, "\n");
	for(i := 0; i < len t.rows; i++) {
		for(j := 0; j < len t.types; j++) {
			if(t.types[j])
				sys->fprint(fd, "\t%d", t.rows[i][j].t1);
			else
				sys->fprint(fd, "\t%s", t.rows[i][j].t0);
		}
		sys->fprint(fd, "\n");
	}
	return nil;
}


Table.text(t: self ref Table): string
{
	s := sprint("%s:\n\tdescr \"%s\"\n", t.name, t.descr);

	if(t.match != nil)
		s += "\tmatch "+t.match.text()+"\n";

	for(l := t.per; l != nil; l = tl l)
		s += "\tper "+hd l+"\n";

	for(l = t.keep; l != nil; l = tl l)
		s += "\tkeep "+hd l+"\n";

	for(ff := t.final; ff != nil; ff = tl ff)
		s += "\t"+(hd ff).text()+"\n";

	return s;
}

Expr.text(e: self ref Expr): string
{
	pick ee := e {
	And =>
		return "("+ee.p0.text()+" and "+ee.p1.text()+")";
	Or =>
		return "("+ee.p0.text()+" or "+ee.p1.text()+")";
	Eq =>
		return "("+ee.p0.text()+" == "+ee.p1.text()+")";
	Ne =>
		return "("+ee.p0.text()+" != "+ee.p1.text()+")";
	Param =>
		return ee.s;
	Str =>
		return "\""+ee.s+"\"";
	}
}

Final.text(f: self ref Final): string
{
	pick ff := f {
	Sort =>
		return sprint("sort %s", ff.s);
	Limit =>
		return sprint("limit %d", ff.n);
	Reverse =>
		return "reverse";
	Normalise =>
		return sprint("normalise %s", ff.s);
	Lump =>
		return sprint("lump %s %s \"%s\"", ff.s, ff.cutoff, ff.str);
	}
}

has(s: string, l: list of string): int
{
	for(; l != nil; l = tl l)
		if(hd l == s)
			return 1;
	return 0;
}

l2a[T](l: list of T): array of T
{
	a := array[len l] of T;
	i := 0;
	for(; l != nil; l = tl l)
		a[i++] = hd l;
	return a;
}

say(s: string)
{
	if(dflag)
		warn(s);
}

warn(s: string)
{
	sys->fprint(sys->fildes(2), "%s\n", s);
}

fail(s: string)
{
	warn(s);
	raise "fail:"+s;
}

	ircfs.dis\
	itest.dis\
	itreplay.dis\
	kill.dis\
	listen.dis\
	lockfs.dis\
	logfile.dis\
	#logparse.dis\
	look.dis\
	ls.dis\
	lstar.dis\
	lszip.dis\
	m4.dis\
	man2html.dis\
	man2txt.dis\

DISBIN=$ROOT/dis

<$ROOT/mkfiles/mkdis
<$ROOT/mkfiles/mksubdirs

auhdr.dis: auplay.dis
	rm -f auhdr.dis && cp auplay.dis auhdr.dis
logparse.b: logparse.y
	yacc -o logparse.b logparse.y

dbfs.dis: $MODDIR/styxservers.m
rawdbfs.dis: $MODDIR/styxservers.m
import.dis:	$MODDIR/encoding.m $MODDIR/factotum.m
basename.dis: $MODDIR/names.m
cleanname.dis: $MODDIR/names.m
vacfs.dis:	$MODDIR/vac.m $MODDIR/venti.m
vacget.dis:	$MODDIR/vac.m $MODDIR/venti.m
vacput.dis:	$MODDIR/vac.m $MODDIR/venti.m

	keyset.dis\
	libc.dis\
	libc0.dis\
	lists.dis\
	lock.dis\
	login.dis\
	man.dis\
	mhttp.dis\
	memfs.dis\
	mpeg.dis\
	msgio.dis\
	#mercurial.dis\
	#mercurialwire.dis\
	#mercurialremote.dis\
	bdiff.dis\

#!/dis/sh
/dis/ip/shttpd -h -n /services/httpd/config.db

<../../mkconfig

DIRS=\

	webget\

SHTARG=\
	auth.sh\
	httpd.sh\
	net.sh\
	registry.sh\
	rstyx.sh\
	styx.sh\

BIN=$ROOT/dis/svc

window: ref Tk->Toplevel;

W: adt {
	textwidth: fn(nil: self ref W, text: Text): int;
};

# TODO theme support
ROMAN: con "/fonts/hermit/regular/16/font";
BOLD: con "/fonts/hermit/bold/16/font";
ITALIC: con "/fonts/hermit/regularitalic/16/font";
HEADING1: con "/fonts/hermit/bold/16/font";
HEADING2: con "/fonts/hermit/bolditalic/16/font";
rfont, bfont, ifont, h1font, h2font: ref Font;

GOATTR: con Parseman->ATTR_LAST << iota;
MANPATH: con "/man/1/man";
INDENT: con 40;

metrics: Parseman->Metrics;

	for (i := 0; i < len shwin_cfg; i++)
		cmd(t, shwin_cfg[i]);
	(menuw, nil) := itemsize(t, ".m");
	if (font != nil) {
		if (font[0] != '/' && (len font == 1 || font[0:2] != "./"))
			font = "/fonts/" + font;
		cmd(t, ".ft.t configure -font " + font);
	} else { # TODO theme support
		font = "/fonts/BQN386/16/font";
		cmd(t, ".ft.t configure -font " + font);
	}
	cmd(t, ".ft.t configure -width 65w -height 20h");
	cmd(t, "pack propagate . 0");
	if(width != nil)
		cmd(t, ". configure -width " + width);
	if(height != nil)
		cmd(t, ". configure -height " + height);

#!/dis/sh
/dis/ip/shttpd -h -n /services/httpd/config.db

message headers (also other protocols than http):
http://www.iana.org/assignments/message-headers/message-header-index.html

note: this document is not finished yet

# introduction

this document contains comments on the http/1.1 specification, as in rfc2616.  it also attempts to provide pointers for implementors, by showing common pitfalls, minimum requirements for implementations (and parts of the specification that can be skipped while remaining compliant), problems/ambiguities in the specification, examples.  this is not (yet) a full introduction to http/1.1, it may become one in the future.

first an example of a contemporary http/1.1 transaction.  sent from a firefox 2.0.0.x client to a lighttpd 1.4.x server.  first the request:

	GET / HTTP/1.1
	Host: www.ueber.net
	User-Agent: Mozilla/5.0 (X11; U; OpenBSD i386; en-US; rv:1.8.1.6) Gecko/20070819 Firefox/2.0.0.6
	Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
	Accept-Language: en-us,en;q=0.5
	Accept-Encoding: gzip,deflate
	Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
	Keep-Alive: 300
	Connection: keep-alive
	

and the response:

	HTTP/1.1 200 OK
	Content-Type: text/html; charset=utf-8
	ETag: "6646323153445497512"
	Accept-Ranges: bytes
	Last-Modified: Sat, 24 Feb 2007 16:35:45 GMT
	Content-Length: 590
	Date: Fri, 14 Dec 2007 13:38:16 GMT
	Server: lighttpd/1.4.13
	
	[590 bytes of body]


as for the request.  the first line is a method (`GET`), a path (`/`) and the http version (`HTTP/1.1`).  each separated by a single space.  lines end with CRLF, but servers should be permissive and accept a LF instead of CRLF as well.  next follow the headers, typically one per line, the colon separates the name from the value.  practically always, leading and trailing whitespace can be removed.  an empty line ends the headers.  if a body is present it follows the empty line immediately.  the length of the body is determined by the headers.

the response is very similar.  only the first line is different:  it consists of the version (`HTTP/1.1`), followed by a 3-digit status code (`200`), followed by a descriptive message (`OK`).  the message leads to the end of the line and is allowed to contain spaces.  the three elements are again separated by a single space.  the body follows the empty line after the headers immediately, just like with a request.  some headers can occur only in requests, some only in responses, some in both.  some headers that are allowed in both have differing semantics based on whether they are used in a request or response.

the most common requests use the methods `GET` and `POST`.  other methods, namely `HEAD`, `OPTIONS` and `TRACE` are used much less.  `PUT` and `DELETE` are not normally used in browsers, browsers just download things.  more methods have been specified for e.g. WEBDAV.  `PUT` and `DELETE` are mostly used with WEBDAV by the way.  http/1.1 also specifies `CONNECT`, which is a bit special:  it makes the http proxy/server forward the current tcp connection to some random ip:port;  used for doing the secure "https" through a proxy.

in general, http gets complex very quickly due to the state-explosive combinations of requests, response statuses and the many headers.  the specification tries to describe behaviour such that new methods and headers can be specified.  unfortunately, new methods and headers can easily introduce inconsistencies.

those who consider a solution to many problems, please read the following rfcs and documents:

* rfc3143, "Known HTTP Proxy/Caching Problems"
* rfc3205, "On the use of HTTP as a Substrate"
* http://www.usenix.org/events/usenix99/invited_talks/mogul.pdf, "What’s wrong with HTTP (and why it doesn't matter)"

or in short: try to avoid http where you can.


# on http headers in general

headers are of the form `field: value`.  whitespace (space/tab, or crlf followed by at least one space/tab) can, but does not have to occur before the value.  so these are all equivalent:

	Content-Length:<sp>123<crlf>
	Content-Length:123<sp><tab><crlf>
	Content-Length:<sp><tab><sp>123<crlf>
	Content-Length: <crlf>
	<sp><tab>123<sp><sp><crlf>

in general, whitespace can occur between tokens that make up a header, unless specically specified as not allowed.  some headers where this is not allowed: `Date` (precisely specified), `Content-Type` (actually, only the media-type in the content type).  whitespace has no meaning, it just serves to separate tokens and readability.

headers aren't really hard, although the way they are specified leaves it a bit unclear how complext it is going to get.  for example, section 2.2 "basic rules" specify `comments` for headers, i.e. data with parentheses.  and comments can be nested.  parsing this with simple string functions isn't nice.  luckily, after reading the specification you'll know that only a few headers are allowed to have headers (in others, the parenteses are simply part of the value).  the headers are: `Via`, `Server`, `User-Agent`.  all headers for which the value doens't have important semantics.

then about the headers whose values consists of comma-separated elements.  these are specified in bnf as `#rule`, meaning zero or more `rule`'s separated by comma's (and lws in between).  the special trick is that multiple headers with the same name may be present, and should be concatenated (with a comma as separator) when handling that header.  about duplicate header names for other headers the specification is silent.  it may be a good idea to treat duplicate headers with known semantics which don't consist of comma-separated values as invalid (i.e. respond with 400 "bad request" if a server encounters it in a request).  the order in which the headers occur is important.  the `#rule`-headers are: `Accept`, `Accept-Charset`, `Accept-Encoding`, `Accept-Language`, `Allow`, `Cache-Control`, `Connection`, `Content-Encoding`, `Content-Language`, `Expect`, `Pragma`, `Proxy-Authenticate`, `TE`, `Trailer`, `Transfer-Encoding`, `Upgrade`, 

note that some headers partially consist of `#rule`-values, but may (partially) consist of other values too.  these are not allowed to be merged.  these headers are: `Accept-Ranges`, `If-Match`, `If-None-Match`, `Vary`, `Via`, `Warning`, `WWW-Authenticate`.

some header values can be "quoted-strings": double quotes around any character except controls and the double quote.  in http/1.0 that was it.  in http/1.1 quoted-string parsing has changed, a backslash is now allowed to escape a double quote.  thus, interpretation of headers should be done based on the http version of the request.


# connections

keeping connections alive after a request is an important feature in http/1.1 compared to http/1.0.  by default, http/1.1 connections are keep-alive connections.  so, after a request another request can follow.  when either the request or the response contains a `Connection: close` header, the connection will close after the response has come in.  


# entity

a _request_ has an entity when either a `Content-Length` or `Transfer-Encoding` header is present.

whether a _response_ has an entity is based on request method and response status code.  section 4.3 "message body" tells us responses to a `HEAD` request do never have bodies, even if there are headers such as `Content-Length` that suggest otherwise.  the section also tells all 1xx, 204 "no content" and 304 "not modified" responses to not have bodies.  all other responses do have bodies (though maybe of zero length).

for requests, only POST and PUT normally have bodies.  OPTIONS is allowed to have a body has well, for future compatibility.  it's semantics are undefined, so it is best not used.  rfc2616 suggests that such a body is discarded.  this is best done by simply closing the connection after reading having sent the response.  this prevents the incoming data to be regarded as a request.

now for the length of the entity if it is present.  first, if `Transfer-Encoding` has a value other than `identity`, the response is in chunked mode, which is self-delimiting.  if a `Content-Length` is present, it is the length in bytes.  note that both headers should not be present in a single request/response;  if they are, `Content-Length` must be ignored.  this strikes me as odd:  if `Transfer-Encoding: identity` is present, how would you determine the length?  next section 4.4 "message length" describes that a `Content-Type` with media-type `multipart/byteranges` is self-delimiting and thus doesn't need a `Transfer-Encoding` or `Content-Length`.  they could have just required chunked transfer-encoding for that case... the last way of determining the length, only when none of the just mentioned methods apply, is to simply read to end of file.  this is mostly for http/1.0 backwards compatibility but is also used between http/1.1-only transactions.  actually, this last method is quite unsatisfactory:  it doesn't allow detecting if a remote server is shutdown (causing its tcp connections to close, i.e. without timing out).


# forward compatibility

rfc2616 sprinkles information on how unrecognized http versions, methods, response statuses, header names and header values should be handled throughout the specification.  the goal is to allow future http versions/extensions to be specified without breaking older clients.  this section contains this data.

1. unknown http versions (section 3.1 "http version").  a higher minor version implies newer message semantics but **not** new message parsing algorithms.  i.e. the new version may have new headers.  the new version will probably be specified such that http/1.1 interpreters of the message will stuff handle the message correctly.  a higher major version implies new message parsing semantics.  thus, if the server is even able to detect the http version in the message with a new major version (a new specification will probably make sure it is), it cannot assume it correctly parsed the message and thus must respond with 505 "http version not supported".  a request in unknown minor version should probably be handled as the highest known minor version understood.  care must be taken to respond properly to unknown values in headers.  the specification explicitly defines what to do for some headers.  unfortuantely, in practice all bets are probably off:  the http/1.0 specification prescribes practically the same, but still http/1.1 messages are parsed differently (chunked encoding isn't present in http/1.0) and the major version number has not been incremented.

2. unknown http methods (section 5.1.1, "method").  methods that are specified but not implemented (e.g. `PUT`), or not completely unrecognized must be responded to with 501 "not implemented".

3. unknown http header names.  this one is easy, unrecognized names should simply be ignored.  this is an explicit forward-compatibility thing.

4. unknown http header values.  this is one is much more tricky.  some headers with invalid values should be ignored.  xxx should some be responded to with a "bad request"?  for some header names a few values are specified and extension values are explicitly allowed in the BNF.  how to handle these unknown values varies.  how invalid values should be handled isn't always clear as well.  the next listing attempts to provide sane defaults.

	* `If-Modified-Since` and `If-Unmodified-Since`: if the date is invalid, it should be ignored.  i think 400 "bad request" would have been better.
	* `Cache-Control` explictly allows extensions.  they are supposedly designed such that they can be ignored.  the extension is supposed to override the default caching directives that apply or are specified in the header for backwards compatibility.  too bad, cache-control is already too complex.
	* `Content-Encoding` implies a 415 "unsupported media type" should be sent for unknown encodings.  unknown encodings aren't really allowed, all valid encodings are specified in section 3.5.
	* `Content-Length`, although 14.13 doesn't say what to do for invalid values, the only proper interpration is "bad request" or "bad response".
	* `Content-Type`, again, sections 14.17 and 7.2.1 do not specify how to handle invalid values.  if the content-type is not specified, it can safely be assumed to be application/octet-stream (which can be responded to with 415 "unsupported media type").  if the value is syntactically invalid, it should probably be treated as a 400 "bad request".
	* `Expect`:  the only value specified is `100-continue`.  if the value is unknown, an error should be returned according to section 14.20.  400 "bad request" seems appropriate.  if the value is known but cannot be met, 417 "expectation failed" applies.
	* `Host`: this header must be present in all requests.  the really silly thing:  if the host is not a valid host on the server (i.e. the server doesn't know the host), a 400 "bad request" must be returned.  404 "file not found" sounds so much better, there is nothing "bad" about the request.
	* `If-Match`: section 14.24, typically used to safely overwrite a file with a `PUT` request.  bad values should be responded to with 400 "bad request" to be safe.  The same goes for `If-Modified-Since` from section 14.28.
	* `Transfer-Encoding`:  section 14.41.  again, nothing is said about unknown encodings.  however, if you don't understand how a message if formed, the only useful action seems to be not to interpret it.  400 "bad request" probably comes closest.
	* `Vary`:  section 14.44, again nothing specified about invalid values.  the safe option seems to treat the value as `*`, the most restrictive and safe value.
	* many other headers are optional.  e.g. `Date`, `ETag`, `If-None-Match, `Last-Modified` only provide additional services (caching).  `Range` provides partial file serving.  if they are invalid, they are safe to igore.  in my opinion it is usually better to point the program to its error rather than guessing what the programs intentions were, but that is not what the specification says.

# minimal implementation & http in practice

bad for us:  the http/1.1 specification is pretty large.
good for us:  we can safely ignore parts of it.  and some parts are only for clients, proxies and/or servers.

as for methods, only `GET` and `HEAD` are mandatory.  a webserver without `POST` is crippled.  `OPTIONS` and `TRACE` aren't very useful but aren't that hard to implement either.

## ignorable headers

we can cut a lot more in headers we need to support.  first, headers that are completely irrelevant to servers:  `Age`, `Max-Forwards`, `Proxy-Authenticate`, `Via`, `Warning`.  these are only for caches (e.g. proxies).

next, some incoming headers have no influence on responses and can safely be ignored: `From`, `Referer`, `User-Agent`.  these may be useful for logging in their raw form.

now, yet another set of headers "should" or "may" be handled, but are not (useful) in practice: `Accept`, `Accept-Charset`, `Accept-Encoding`, `Accept-Language`, `Content-Location`, `Content-MD5`, `TE`, `Upgrade`.

## mandatory/useful headers

then, another set of headers are only generated by servers, not received.  generating headers is usually far easier to implement than parsing them.  they are:  `Allow`, `Accept-Ranges`, `ETag`, `Expires`, `Last-Modified`, `Location`, `Server`, `Vary`, `WWW-Authenticate`.  these headers can be generated by both clients and servers, but the server can ignore them when they come from a client:  `Date`, `Cache-Control`, `Pragma`, `Trailer`.

headers that clients send but can safely be ignored, but are quite useful and recommended to be implemented: `Authorization`, `If-Modified-Since`, `If-None-Match`, `If-Range`, `Range`.  note: `Content-Range` is also only generated by a server, though not entirely trivial.  it is a response to the `Range` header.

headers that clients send and really really should be implemented (perhaps partially): `Connection`, `Content-Encoding`, `Content-Length`, `Content-Type`, `Expect`, `Host`, `If-Match`, `If-Unmodified-Since`, `Transfer-Encoding`.

for security, some headers should be handled specially.  when a `Proxy-Authorization` is sent to a server it should respond with 400 "bad request", this should just never happen.  when a `Authorization` header is sent for a path on a server that doesn't need authentication, it should only be accepted when the user/pass are empty strings;  for anything else, a 401 "unauthorized" should be send.  note that is not specified by the specification but should protect users from spewing their credentials around.

# on specific headers

## `Connection`

if one of the values (it is a comma-separated list) is `close`, the connection will be closed.  the response should also include a `Connection: close` header.  other values can safely be ignored.

## `Content-Encoding`

the server can generated this, e.g. for sending compressed data.  the client can send this for a `POST` too, though it doesn't seem to be common practice.  there is no good reason it isn't though.

## `Content-Length`

if present, and `Transfer-Encoding` is not present, this denotes the size of the message body.  if `Transfer-Encoding` is present (they shouldn't both be), `Content-Length` should be ignored.

## `Content-Type`

it should be present whenever an entity is sent.  if it isn't, `application/octet-stream` may be assumed.  otherwise, remember to specify a decent `charset` for the data if it is `text/*`, e.g.: `Content-Type: text/html; charset=utf-8`.

## `Expect`

only relevant for `POST`.  if the value is not (case-insensitive) `100-continue`, or if it is but the request is not acceptable, send 417 "expectation failed".  note that `100-continue` will usually have to be handled by the programs that handle the `POST` request, which is usually a separate process started by the webserver.

## `Host`

_All_ requests must have a `Host` header (see section 9).  Even if it isn't used, e.g. for `TRACE` requests.  Also, when the path, specified after the method, is a full URL, the `Host` must still be present but must also be ignored.

## `If-Match` and `If-Unmodified-Since`

only mandatory for `PUT` and `DELETE`, to make operations safe.  are not really useful for `GET` etc.

## `Transfer-Encoding`

when present, in practice it has the value `chunked`.  this one is "funny" to read about in rfc2616.  first, according to section 14.41 multiple transformations can be applied to message.  thus, we are allowed to send `Transfer-Encoding: chunked, chunked`, and wrap one chunk-stream in another.  utterly silly and annoying.  xxx check how many clients support this.

on the other hand:  section 4.4, list item 2, says that if the value is anything other than `identity` it means it is simply chunked.  though that section implies something other hilarious again.

***

## `Allow`

send this comma-separated value in response to an `OPTIONS` request.  not much use sending it otherwise, though you are allowed to.

## `Accept-Ranges`

you can send this with value `bytes` if you support range requests (with the `Range` request-header and `Content-Range` response-header).  it isn't required though, clients can always try to send range requests.  it is probably useful to send this for `OPTIONS` requests.

## `ETag`

these are only mandatory for `Range` requests.  they are very useful for caching since they specify the version of a file uniquely.  the older (and seemingly more popular, even in this century) habit of using timestamps with second-precision has only disadvantages.

etags are simply quoted strings that represent the entity uniquely.  the SHA-1 hash of the contents can be used, but this usually isn't feasible.  better:  the SHA-1 hash of the concatenatino of: last modification time (of the file or of the data generated content is based on), requested URL (host, port, path, query), file modification count.

the etag should normally be about the entity returned (e.g. with a 200 "OK" response) or not returned (e.g. with a 304 "not modified" response).  for 201 "created", the etag is about the object created, not the response.  it doesn't make much sense to return etags in error responses, such as 404 "file not found", they would have to be about the error message, which isn't useful.

## `Expires`

used for specifying how long an entity can be cached.  data that should not be cached can send a date string in the past, or equal to the current date minus approximately a year (this comes from the specification indeed..., section 14.21).  but that's mostly for compatibility with http/1.0.  sending a `Cache-Control: max-age=0` is preferred (by me at least).

## `Last-Modified`

the last modification time of the file or data the entity is based on, as a date string.  the value is again used by clients in conditional requests, with the `If-Unmodified-Since` header.  there is no reason not to include it when an mtime is available.

## `Location`

an url pointing to an object.  should be present in: 301 "moved permanently", 302 "found", 303 "see other", 307 "temporary redirect";  may be present in: 300 "multiple choices"

a 305 "use proxy" should have a location header pointing to a proxy.  this response doesn't make much sense though.

in practice, just implement for the 3xx responses mentioned above.

## `Server`

an identification of the server.  is not mandatory, but useful for debugging and statistics.  note that this isn't free-form, use `name/version`, e.g. `myhttpd/1`.

## `Vary`

used for specifying (to a cache) which headers were used in content negotiation.  if the value is `*`, other elements than headers have influenced the representation chosen.  otherwise, it is a comma-separated list of header fields.  a cache may return a previously returned response if the headers listed are identical and the other caching headers are valid.

## `WWW-Authenticate`

must be present in a 401 "unauthorized".  specified in rfc 2617.  "basic" authorization just sends plaintext username/passwords.  "digest" authorization uses a challenge.  not great.

***

## `Date`

the date the client/server created the request/response.  no reason not to include it.

## `Cache-Control`

this header as many pages in the rfc describing its behaviour.  if you want things explicitly uncacheable, make the server return `max-age=0`.  safe to ignore otherwise, mostly relevant for caches.

## `Pragma`

for backwards compatibility.  http/1.0 used this for caching.  when making an object explicitly uncacheable, set it to `no-cache`.  safe to ignore otherwise.

## `Trailer`

comma-separated list of headers that will be present in the trailing headers after the chunk-encoded stream.  there is rarely a reason to need and implement this.  trailing headers must be explicitly allowed.  safe to ignore.


# on http quirks

this section describes some strange things about http/1.1.

servers should ignore empty lines at the beginning of a request.  for being lenient towards crappy clients.

quality values in headers, e.g. `q=0.2` are a bit silly.  e.g. these are all valid values: `0.`, `0.0`, `0`, `0.000`.  do they all have the same semantics?  elsewhere in the specification, special meaning is assigned to a value of `0`.  do they mean the floating point value zero?  simply picking numbers from 0-1000 would have been easier.

needing to understand three forms of dates...

full url should be understood by servers, but Host: must always be present.  if a full url is present, the Host header must simply be ignored.

a Host: header should include the port the request came in on.  this is useless, the server can always determine which port it was.  a client can lie about this.  the hostname is different of course, that's what the host header is for.

section 4.4, list item 2 claims that a message has `Transfer-Encoding` with a value other than `identity`, it means the transfer encoding is chunked, "unless the message is terminated by closing the connection".  this implies a response with such a header value will have to be treated as chunked and buffered somewhere, and if the connection ends before the proper end-of-chunked-stream message has come in, the entire chunked stream (with chunk messages) will retroactively have to be interpreted as a literal stream.  hilarious.  sad.

a cache is only allowed to combine range requests if the etag is strongly equivalent.  sure.  a client, however, is not granted the same protection, it is allowed (well, not forbidden as with caches) to combine the responses.  it doesn't make sense to specify this restriction just for caches.

the whole idea of using timestamps with second-precision for caching isn't great.  the way the timestamp is specified is even more ridiculous.  the fact that entities can be conditionally retrieved based on the timestamp is even more silly given that etags are so much more useful for that.

an `OPTIONS` request wants to know which methods are allowed on an object.  to determine this, we may need authorization credentials.  otherwise we might be giving away information the client is not allowed to know.  and besides, it sounds pretty bloaty...  so instead, we'll just say we support POST on everything, as if `*` was specified for path.  also, it seems `OPTIONS` is somewhwat underspecified.  e.g., what to do when the path does not exist?  lastly, no one seems to use `OPTIONS`, so we really do not care enough.

# on http/1.0 compatibility

xxx to write
mention headers new to http/1.1, which must be ignored for http/1.0.
mention new status response codes and when they are used.
new methods?  aren't defined right?

http/1.0 headers:

- Allow
- Authorization
- Content-Encoding
- Content-Length
- Content-Type
- Date
- Expires
- From
- If-Modified-Since
- Last-Modified
- Location
- Pragma
- Referer
- Server
- User-Agent
- WWW-Authenticate

new in http/1.1:

- Accept
- Accept-Charset
- Accept-Encoding
- Accept-Language
- Accept-Ranges
- Age
- Cache-Control
- Connection
- Content-Language
- Content-Location
- Content-MD5
- Content-Range
- ETag
- Expect
- Host
- If-Match
- If-None-Match
- If-Range
- If-Unmodified-Since
- Max-Forwards
- Proxy-Authenticate
- Proxy-Authorization
- Range
- Retry-After
- TE
- Trailer
- Transfer-Encoding
- Upgrade
- Vary
- Via
- Warning

sending headers such as `ETag` to a http/1.0 client is not a problem, clients should not do anything with them.  some incoming headers should really be handled for http/1.1 only:

- Expect
- If-Match
- If-None-Match
- If-Range
- If-Unmodified-Since
- Range
- Transfer-Encoding

missing length for clients sending a body must result in 400, not 411? (rfcrfc1945#7.2.2)
if-modified-since should be ignored for head request (rfc1945#8.1)