Start refactoring towards netlib, adding SNI and client testing.

This commit is contained in:
Aldo Cortesi 2012-06-17 07:57:24 +12:00
parent b1f410c78d
commit bd99a13f39
6 changed files with 514 additions and 316 deletions

182
libpathod/netlib.py Normal file
View File

@ -0,0 +1,182 @@
import select, socket, threading, traceback, sys
from OpenSSL import SSL
class NetLibError(Exception): pass
class FileLike:
def __init__(self, o):
self.o = o
def __getattr__(self, attr):
return getattr(self.o, attr)
def flush(self):
pass
def read(self, length):
result = ''
while len(result) < length:
try:
data = self.o.read(length)
except SSL.ZeroReturnError:
break
if not data:
break
result += data
return result
def write(self, v):
self.o.sendall(v)
def readline(self, size = None):
result = ''
bytes_read = 0
while True:
if size is not None and bytes_read >= size:
break
ch = self.read(1)
bytes_read += 1
if not ch:
break
else:
result += ch
if ch == '\n':
break
return result
class TCPClient:
def __init__(self, ssl, host, port, clientcert):
self.ssl, self.host, self.port, self.clientcert = ssl, host, port, clientcert
self.connection, self.rfile, self.wfile = None, None, None
self.cert = None
self.connect()
def connect(self):
try:
addr = socket.gethostbyname(self.host)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.ssl:
context = SSL.Context(SSL.SSLv23_METHOD)
if self.clientcert:
context.use_certificate_file(self.clientcert)
server = SSL.Connection(context, server)
server.connect((addr, self.port))
if self.ssl:
self.cert = server.get_peer_certificate()
self.rfile, self.wfile = FileLike(server), FileLike(server)
else:
self.rfile, self.wfile = server.makefile('rb'), server.makefile('wb')
except socket.error, err:
raise NetLibError('Error connecting to "%s": %s' % (self.host, err))
self.connection = server
class BaseHandler:
rbufsize = -1
wbufsize = 0
def __init__(self, connection, client_address, server):
self.connection = connection
self.rfile = self.connection.makefile('rb', self.rbufsize)
self.wfile = self.connection.makefile('wb', self.wbufsize)
self.client_address = client_address
self.server = server
self.handle()
self.finish()
def convert_to_ssl(self, cert, key):
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.use_privatekey_file(key)
ctx.use_certificate_file(cert)
self.connection = SSL.Connection(ctx, self.connection)
self.connection.set_accept_state()
self.rfile = FileLike(self.connection)
self.wfile = FileLike(self.connection)
def finish(self):
try:
if not getattr(self.wfile, "closed", False):
self.wfile.flush()
self.connection.close()
self.wfile.close()
self.rfile.close()
except IOError: # pragma: no cover
pass
def handle(self): # pragma: no cover
raise NotImplementedError
class TCPServer:
request_queue_size = 20
def __init__(self, server_address):
self.server_address = server_address
self.__is_shut_down = threading.Event()
self.__shutdown_request = False
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(self.server_address)
self.server_address = self.socket.getsockname()
self.socket.listen(self.request_queue_size)
self.port = self.socket.getsockname()[1]
def request_thread(self, request, client_address):
try:
self.handle_connection(request, client_address)
request.close()
except:
self.handle_error(request, client_address)
request.close()
def serve_forever(self, poll_interval=0.5):
self.__is_shut_down.clear()
try:
while not self.__shutdown_request:
r, w, e = select.select([self.socket], [], [], poll_interval)
if self.socket in r:
try:
request, client_address = self.socket.accept()
except socket.error:
return
try:
t = threading.Thread(
target = self.request_thread,
args = (request, client_address)
)
t.setDaemon(1)
t.start()
except:
self.handle_error(request, client_address)
request.close()
finally:
self.__shutdown_request = False
self.__is_shut_down.set()
def shutdown(self):
self.__shutdown_request = True
self.__is_shut_down.wait()
self.handle_shutdown()
def handle_error(self, request, client_address, fp=sys.stderr):
"""
Called when handle_connection raises an exception.
"""
print >> fp, '-'*40
print >> fp, "Error processing of request from %s:%s"%client_address
print >> fp, traceback.format_exc()
print >> fp, '-'*40
def handle_connection(self, request, client_address): # pragma: no cover
"""
Called after client connection.
"""
raise NotImplementedError
def handle_shutdown(self):
"""
Called after server shutdown.
"""
pass

263
libpathod/opathod.py Normal file
View File

@ -0,0 +1,263 @@
import urllib, pprint
import tornado.web, tornado.template, tornado.ioloop, tornado.httpserver
import rparse, utils, version
class Pathod(object):
def __init__(self, spec, application, request, **settings):
self.application, self.request, self.settings = application, request, settings
try:
self.response = rparse.parse(self.settings, spec)
except rparse.ParseException, v:
self.response = rparse.InternalResponse(
800,
"Error parsing response spec: %s\n"%v.msg + v.marked()
)
def _execute(self, transforms, *args, **kwargs):
d = self.response.serve(self.request)
d["request"] = dict(
path = self.request.path,
method = self.request.method,
headers = self.request.headers,
host = self.request.host,
protocol = self.request.protocol,
remote_address = self.request.connection.address,
full_url = self.request.full_url(),
query = self.request.query,
version = self.request.version,
uri = self.request.uri,
)
self.application.add_log(d)
class RequestPathod(Pathod):
anchor = "/p/"
def __init__(self, application, request, **settings):
spec = urllib.unquote(request.uri)[len(self.anchor):]
Pathod.__init__(self, spec, application, request, **settings)
class PathodApp(tornado.web.Application):
LOGBUF = 500
def __init__(self, **settings):
self.appsettings = settings
tornado.web.Application.__init__(
self,
[
(r"/", Index),
(r"/log", Log),
(r"/log/clear", ClearLog),
(r"/log/([0-9]+)", OneLog),
(r"/help", Help),
(r"/preview", Preview),
(r"/api/shutdown", APIShutdown),
(r"/api/info", APIInfo),
(r"/api/log", APILog),
(r"/api/log/clear", APILogClear),
(r"/p/.*", RequestPathod, settings),
],
static_path = utils.data.path("static"),
template_path = utils.data.path("templates"),
debug=True
)
self.log = []
self.logid = 0
def add_anchor(self, pattern, spec):
"""
Anchors are added to the beginning of the handlers.
"""
# We assume we have only one host...
l = self.handlers[0][1]
class FixedPathod(Pathod):
def __init__(self, application, request, **settings):
Pathod.__init__(self, spec, application, request, **settings)
FixedPathod.spec = spec
FixedPathod.pattern = pattern
l.insert(0, tornado.web.URLSpec(pattern, FixedPathod, self.appsettings))
def get_anchors(self):
"""
Anchors are added to the beginning of the handlers.
"""
l = self.handlers[0][1]
a = []
for i in l:
if i.handler_class.__name__ == "FixedPathod":
a.append(
(
i.handler_class.pattern,
i.handler_class.spec
)
)
return a
def remove_anchor(self, pattern, spec):
"""
Anchors are added to the beginning of the handlers.
"""
l = self.handlers[0][1]
for i, h in enumerate(l):
if h.handler_class.__name__ == "FixedPathod":
if (h.handler_class.pattern, h.handler_class.spec) == (pattern, spec):
del l[i]
return
def add_log(self, d):
d["id"] = self.logid
self.log.insert(0, d)
if len(self.log) > self.LOGBUF:
self.log.pop()
self.logid += 1
def log_by_id(self, id):
for i in self.log:
if i["id"] == id:
return i
def clear_log(self):
self.log = []
def get_log(self):
return self.log
def make_app(staticdir=None, anchors=()):
"""
staticdir: A directory for static assets referenced in response patterns.
anchors: A sequence of strings of the form "pattern=pagespec"
"""
settings = dict(
staticdir=staticdir
)
application = PathodApp(**settings)
for i in anchors:
rex, spec = utils.parse_anchor_spec(i, settings)
application.add_anchor(rex, spec)
return application
def make_server(application, port, address, ssl_options):
"""
Returns a (server, port) tuple.
The returned port will match the passed port, unless the passed port
was 0. In that case, an arbitrary empty port will be bound to, and this
new port will be returned.
"""
http_server = tornado.httpserver.HTTPServer(
application,
ssl_options=ssl_options
)
http_server.listen(port, address)
port = port
for i in http_server._sockets.values():
sn = i.getsockname()
if sn[0] == address:
port = sn[1]
return http_server, port
# begin nocover
def run(server):
tornado.ioloop.IOLoop.instance().start()
server.stop()
class APILog(tornado.web.RequestHandler):
def get(self):
self.write(
dict(
d = self.application.get_log()
)
)
class APILogClear(tornado.web.RequestHandler):
def post(self):
self.application.clear_log()
self.write("OK")
class APIShutdown(tornado.web.RequestHandler):
def post(self):
tornado.ioloop.IOLoop.instance().stop()
self.write("OK")
class APIInfo(tornado.web.RequestHandler):
def get(self):
self.write(
dict(
version = version.IVERSION
)
)
class _Page(tornado.web.RequestHandler):
def render(self, name, **kwargs):
tornado.web.RequestHandler.render(self, name + ".html", **kwargs)
class Index(_Page):
name = "index"
section = "main"
def get(self):
self.render(self.name, section=self.section, spec="")
class Preview(_Page):
name = "preview"
section = "main"
SANITY = 1024*1024
def get(self):
spec = self.get_argument("spec", None)
args = dict(
spec = spec,
section = self.section,
syntaxerror = None,
error = None
)
try:
r = rparse.parse(self.application.settings, spec)
except rparse.ParseException, v:
args["syntaxerror"] = str(v)
args["marked"] = v.marked()
return self.render(self.name, **args)
if r.length() > self.SANITY:
error = "Refusing to preview a response of %s bytes. This is for your own good."%r.length()
args["error"] = error
else:
d = utils.DummyRequest()
r.serve(d)
args["output"] = d.getvalue()
self.render(self.name, **args)
class Help(_Page):
name = "help"
section = "help"
def get(self):
self.render(self.name, section=self.section)
class Log(_Page):
name = "log"
section = "log"
def get(self):
self.render(self.name, section=self.section, log=self.application.log)
class OneLog(_Page):
name = "onelog"
section = "log"
def get(self, lid):
l = pprint.pformat(self.application.log_by_id(int(lid)))
self.render(self.name, section=self.section, alog=l, lid=lid)
class ClearLog(_Page):
def post(self):
self.application.clear_logs()
self.redirect("/log")

View File

@ -1,265 +1,14 @@
import urllib, pprint
import tornado.web, tornado.template, tornado.ioloop, tornado.httpserver
import rparse, utils, version
import netlib
class PathodHandler(netlib.BaseHandler):
def handle(self):
print "Here"
class Pathod(object):
def __init__(self, spec, application, request, **settings):
self.application, self.request, self.settings = application, request, settings
try:
self.response = rparse.parse(self.settings, spec)
except rparse.ParseException, v:
self.response = rparse.InternalResponse(
800,
"Error parsing response spec: %s\n"%v.msg + v.marked()
)
def _execute(self, transforms, *args, **kwargs):
d = self.response.serve(self.request)
d["request"] = dict(
path = self.request.path,
method = self.request.method,
headers = self.request.headers,
host = self.request.host,
protocol = self.request.protocol,
remote_address = self.request.connection.address,
full_url = self.request.full_url(),
query = self.request.query,
version = self.request.version,
uri = self.request.uri,
)
self.application.add_log(d)
class RequestPathod(Pathod):
anchor = "/p/"
def __init__(self, application, request, **settings):
spec = urllib.unquote(request.uri)[len(self.anchor):]
Pathod.__init__(self, spec, application, request, **settings)
class PathodApp(tornado.web.Application):
LOGBUF = 500
def __init__(self, **settings):
self.appsettings = settings
tornado.web.Application.__init__(
self,
[
(r"/", Index),
(r"/log", Log),
(r"/log/clear", ClearLog),
(r"/log/([0-9]+)", OneLog),
(r"/help", Help),
(r"/preview", Preview),
(r"/api/shutdown", APIShutdown),
(r"/api/info", APIInfo),
(r"/api/log", APILog),
(r"/api/log/clear", APILogClear),
(r"/p/.*", RequestPathod, settings),
],
static_path = utils.data.path("static"),
template_path = utils.data.path("templates"),
debug=True
)
self.log = []
self.logid = 0
def add_anchor(self, pattern, spec):
"""
Anchors are added to the beginning of the handlers.
"""
# We assume we have only one host...
l = self.handlers[0][1]
class FixedPathod(Pathod):
def __init__(self, application, request, **settings):
Pathod.__init__(self, spec, application, request, **settings)
FixedPathod.spec = spec
FixedPathod.pattern = pattern
l.insert(0, tornado.web.URLSpec(pattern, FixedPathod, self.appsettings))
def get_anchors(self):
"""
Anchors are added to the beginning of the handlers.
"""
l = self.handlers[0][1]
a = []
for i in l:
if i.handler_class.__name__ == "FixedPathod":
a.append(
(
i.handler_class.pattern,
i.handler_class.spec
)
)
return a
def remove_anchor(self, pattern, spec):
"""
Anchors are added to the beginning of the handlers.
"""
l = self.handlers[0][1]
for i, h in enumerate(l):
if h.handler_class.__name__ == "FixedPathod":
if (h.handler_class.pattern, h.handler_class.spec) == (pattern, spec):
del l[i]
return
def add_log(self, d):
d["id"] = self.logid
self.log.insert(0, d)
if len(self.log) > self.LOGBUF:
self.log.pop()
self.logid += 1
def log_by_id(self, id):
for i in self.log:
if i["id"] == id:
return i
def clear_log(self):
self.log = []
def get_log(self):
return self.log
def make_app(staticdir=None, anchors=()):
"""
staticdir: A directory for static assets referenced in response patterns.
anchors: A sequence of strings of the form "pattern=pagespec"
"""
settings = dict(
staticdir=staticdir
)
application = PathodApp(**settings)
for i in anchors:
rex, spec = utils.parse_anchor_spec(i, settings)
application.add_anchor(rex, spec)
return application
def make_server(application, port, address, ssl_options):
"""
Returns a (server, port) tuple.
The returned port will match the passed port, unless the passed port
was 0. In that case, an arbitrary empty port will be bound to, and this
new port will be returned.
"""
http_server = tornado.httpserver.HTTPServer(
application,
ssl_options=ssl_options
)
http_server.listen(port, address)
port = port
for i in http_server._sockets.values():
sn = i.getsockname()
if sn[0] == address:
port = sn[1]
return http_server, port
# begin nocover
def run(server):
tornado.ioloop.IOLoop.instance().start()
server.stop()
class APILog(tornado.web.RequestHandler):
def get(self):
self.write(
dict(
d = self.application.get_log()
)
)
class APILogClear(tornado.web.RequestHandler):
def post(self):
self.application.clear_log()
self.write("OK")
class APIShutdown(tornado.web.RequestHandler):
def post(self):
tornado.ioloop.IOLoop.instance().stop()
self.write("OK")
class APIInfo(tornado.web.RequestHandler):
def get(self):
self.write(
dict(
version = version.IVERSION
)
)
class _Page(tornado.web.RequestHandler):
def render(self, name, **kwargs):
tornado.web.RequestHandler.render(self, name + ".html", **kwargs)
class Index(_Page):
name = "index"
section = "main"
def get(self):
self.render(self.name, section=self.section, spec="")
class Preview(_Page):
name = "preview"
section = "main"
SANITY = 1024*1024
def get(self):
spec = self.get_argument("spec", None)
args = dict(
spec = spec,
section = self.section,
syntaxerror = None,
error = None
)
try:
r = rparse.parse(self.application.settings, spec)
except rparse.ParseException, v:
args["syntaxerror"] = str(v)
args["marked"] = v.marked()
return self.render(self.name, **args)
if r.length() > self.SANITY:
error = "Refusing to preview a response of %s bytes. This is for your own good."%r.length()
args["error"] = error
else:
d = utils.DummyRequest()
r.serve(d)
args["output"] = d.getvalue()
self.render(self.name, **args)
class Help(_Page):
name = "help"
section = "help"
def get(self):
self.render(self.name, section=self.section)
class Log(_Page):
name = "log"
section = "log"
def get(self):
self.render(self.name, section=self.section, log=self.application.log)
class OneLog(_Page):
name = "onelog"
section = "log"
def get(self, lid):
l = pprint.pformat(self.application.log_by_id(int(lid)))
self.render(self.name, section=self.section, alog=l, lid=lid)
class ClearLog(_Page):
def post(self):
self.application.clear_logs()
self.redirect("/log")
class PathodServer(netlib.TCPServer):
def __init__(self, addr):
netlib.TCPServer.__init__(self, addr)
def handle_connection(self, request, client_address):
PathodHandler(request, client_address, self)

View File

@ -29,7 +29,6 @@ class PaThread(threading.Thread):
self.q, self.app, self.ssl = q, app, ssl
self.port = None
# begin nocover
def run(self):
if self.ssl is True:
ssloptions = dict(

56
opathod Executable file
View File

@ -0,0 +1,56 @@
#!/usr/bin/env python
import argparse, sys
from libpathod import pathod, utils, version
import tornado.ioloop
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument("-p", dest='port', default=9999, type=int, help='Port. Specify 0 to pick an arbitrary empty port.')
parser.add_argument("-l", dest='address', default="0.0.0.0", type=str, help='Listening address.')
parser.add_argument(
"-a", dest='anchors', default=[], type=str, action="append",
help='Add an anchor. Specified as a string with the form pattern=pagespec'
)
parser.add_argument(
"-d", dest='staticdir', default=None, type=str,
help='Directory for static files.'
)
parser.add_argument(
"-s", dest='ssl', default=False,
action="store_true",
help='Serve with SSL.'
)
parser.add_argument(
"--keyfile", dest='ssl_keyfile', default=None,
type=str,
help='SSL key file. If not specified, a default key is used.'
)
parser.add_argument(
"--certfile", dest='ssl_certfile', default=None,
type=str,
help='SSL cert file. If not specified, a default cert is used.'
)
args = parser.parse_args()
try:
app = pathod.make_app(staticdir=args.staticdir, anchors=args.anchors)
except utils.AnchorError, v:
parser.error(str(v))
sl = [args.ssl_keyfile, args.ssl_certfile]
if any(sl) and not all(sl):
parser.error("Both --certfile and --keyfile must be specified.")
if args.ssl:
ssl = dict(
keyfile = args.ssl_keyfile or utils.data.path("resources/server.key"),
certfile = args.ssl_certfile or utils.data.path("resources/server.crt"),
)
else:
ssl = None
try:
server, port = pathod.make_server(app, args.port, args.address, ssl)
print "%s listening on port %s"%(version.NAMEVERSION, port)
pathod.run(server)
except KeyboardInterrupt:
pass

57
pathod
View File

@ -1,56 +1,5 @@
#!/usr/bin/env python
import argparse, sys
from libpathod import pathod, utils, version
import tornado.ioloop
from libpathod import pathod
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument("-p", dest='port', default=9999, type=int, help='Port. Specify 0 to pick an arbitrary empty port.')
parser.add_argument("-l", dest='address', default="0.0.0.0", type=str, help='Listening address.')
parser.add_argument(
"-a", dest='anchors', default=[], type=str, action="append",
help='Add an anchor. Specified as a string with the form pattern=pagespec'
)
parser.add_argument(
"-d", dest='staticdir', default=None, type=str,
help='Directory for static files.'
)
parser.add_argument(
"-s", dest='ssl', default=False,
action="store_true",
help='Serve with SSL.'
)
parser.add_argument(
"--keyfile", dest='ssl_keyfile', default=None,
type=str,
help='SSL key file. If not specified, a default key is used.'
)
parser.add_argument(
"--certfile", dest='ssl_certfile', default=None,
type=str,
help='SSL cert file. If not specified, a default cert is used.'
)
args = parser.parse_args()
try:
app = pathod.make_app(staticdir=args.staticdir, anchors=args.anchors)
except utils.AnchorError, v:
parser.error(str(v))
sl = [args.ssl_keyfile, args.ssl_certfile]
if any(sl) and not all(sl):
parser.error("Both --certfile and --keyfile must be specified.")
if args.ssl:
ssl = dict(
keyfile = args.ssl_keyfile or utils.data.path("resources/server.key"),
certfile = args.ssl_certfile or utils.data.path("resources/server.crt"),
)
else:
ssl = None
try:
server, port = pathod.make_server(app, args.port, args.address, ssl)
print "%s listening on port %s"%(version.NAMEVERSION, port)
pathod.run(server)
except KeyboardInterrupt:
pass
s = pathod.PathodServer(("127.0.0.1", 8888))
s.serve_forever()