From bd99a13f3965bfdd09a58020424c3b36d97f6877 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 17 Jun 2012 07:57:24 +1200 Subject: [PATCH] Start refactoring towards netlib, adding SNI and client testing. --- libpathod/netlib.py | 182 +++++++++++++++++++++++++++++ libpathod/opathod.py | 263 +++++++++++++++++++++++++++++++++++++++++ libpathod/pathod.py | 271 ++----------------------------------------- libpathod/test.py | 1 - opathod | 56 +++++++++ pathod | 57 +-------- 6 files changed, 514 insertions(+), 316 deletions(-) create mode 100644 libpathod/netlib.py create mode 100644 libpathod/opathod.py create mode 100755 opathod diff --git a/libpathod/netlib.py b/libpathod/netlib.py new file mode 100644 index 000000000..08ccba091 --- /dev/null +++ b/libpathod/netlib.py @@ -0,0 +1,182 @@ +import select, socket, threading, traceback, sys +from OpenSSL import SSL + + +class NetLibError(Exception): pass + + +class FileLike: + def __init__(self, o): + self.o = o + + def __getattr__(self, attr): + return getattr(self.o, attr) + + def flush(self): + pass + + def read(self, length): + result = '' + while len(result) < length: + try: + data = self.o.read(length) + except SSL.ZeroReturnError: + break + if not data: + break + result += data + return result + + def write(self, v): + self.o.sendall(v) + + def readline(self, size = None): + result = '' + bytes_read = 0 + while True: + if size is not None and bytes_read >= size: + break + ch = self.read(1) + bytes_read += 1 + if not ch: + break + else: + result += ch + if ch == '\n': + break + return result + + +class TCPClient: + def __init__(self, ssl, host, port, clientcert): + self.ssl, self.host, self.port, self.clientcert = ssl, host, port, clientcert + self.connection, self.rfile, self.wfile = None, None, None + self.cert = None + self.connect() + + def connect(self): + try: + addr = socket.gethostbyname(self.host) + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if self.ssl: + context = SSL.Context(SSL.SSLv23_METHOD) + if self.clientcert: + context.use_certificate_file(self.clientcert) + server = SSL.Connection(context, server) + server.connect((addr, self.port)) + if self.ssl: + self.cert = server.get_peer_certificate() + self.rfile, self.wfile = FileLike(server), FileLike(server) + else: + self.rfile, self.wfile = server.makefile('rb'), server.makefile('wb') + except socket.error, err: + raise NetLibError('Error connecting to "%s": %s' % (self.host, err)) + self.connection = server + + +class BaseHandler: + rbufsize = -1 + wbufsize = 0 + def __init__(self, connection, client_address, server): + self.connection = connection + self.rfile = self.connection.makefile('rb', self.rbufsize) + self.wfile = self.connection.makefile('wb', self.wbufsize) + + self.client_address = client_address + self.server = server + self.handle() + self.finish() + + def convert_to_ssl(self, cert, key): + ctx = SSL.Context(SSL.SSLv23_METHOD) + ctx.use_privatekey_file(key) + ctx.use_certificate_file(cert) + self.connection = SSL.Connection(ctx, self.connection) + self.connection.set_accept_state() + self.rfile = FileLike(self.connection) + self.wfile = FileLike(self.connection) + + def finish(self): + try: + if not getattr(self.wfile, "closed", False): + self.wfile.flush() + self.connection.close() + self.wfile.close() + self.rfile.close() + except IOError: # pragma: no cover + pass + + def handle(self): # pragma: no cover + raise NotImplementedError + + +class TCPServer: + request_queue_size = 20 + def __init__(self, server_address): + self.server_address = server_address + self.__is_shut_down = threading.Event() + self.__shutdown_request = False + self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.socket.bind(self.server_address) + self.server_address = self.socket.getsockname() + self.socket.listen(self.request_queue_size) + self.port = self.socket.getsockname()[1] + + def request_thread(self, request, client_address): + try: + self.handle_connection(request, client_address) + request.close() + except: + self.handle_error(request, client_address) + request.close() + + def serve_forever(self, poll_interval=0.5): + self.__is_shut_down.clear() + try: + while not self.__shutdown_request: + r, w, e = select.select([self.socket], [], [], poll_interval) + if self.socket in r: + try: + request, client_address = self.socket.accept() + except socket.error: + return + try: + t = threading.Thread( + target = self.request_thread, + args = (request, client_address) + ) + t.setDaemon(1) + t.start() + except: + self.handle_error(request, client_address) + request.close() + finally: + self.__shutdown_request = False + self.__is_shut_down.set() + + def shutdown(self): + self.__shutdown_request = True + self.__is_shut_down.wait() + self.handle_shutdown() + + def handle_error(self, request, client_address, fp=sys.stderr): + """ + Called when handle_connection raises an exception. + """ + print >> fp, '-'*40 + print >> fp, "Error processing of request from %s:%s"%client_address + print >> fp, traceback.format_exc() + print >> fp, '-'*40 + + def handle_connection(self, request, client_address): # pragma: no cover + """ + Called after client connection. + """ + raise NotImplementedError + + def handle_shutdown(self): + """ + Called after server shutdown. + """ + pass diff --git a/libpathod/opathod.py b/libpathod/opathod.py new file mode 100644 index 000000000..6ec8367d3 --- /dev/null +++ b/libpathod/opathod.py @@ -0,0 +1,263 @@ +import urllib, pprint +import tornado.web, tornado.template, tornado.ioloop, tornado.httpserver +import rparse, utils, version + + +class Pathod(object): + def __init__(self, spec, application, request, **settings): + self.application, self.request, self.settings = application, request, settings + try: + self.response = rparse.parse(self.settings, spec) + except rparse.ParseException, v: + self.response = rparse.InternalResponse( + 800, + "Error parsing response spec: %s\n"%v.msg + v.marked() + ) + + def _execute(self, transforms, *args, **kwargs): + d = self.response.serve(self.request) + d["request"] = dict( + path = self.request.path, + method = self.request.method, + headers = self.request.headers, + host = self.request.host, + protocol = self.request.protocol, + remote_address = self.request.connection.address, + full_url = self.request.full_url(), + query = self.request.query, + version = self.request.version, + uri = self.request.uri, + ) + self.application.add_log(d) + + +class RequestPathod(Pathod): + anchor = "/p/" + def __init__(self, application, request, **settings): + spec = urllib.unquote(request.uri)[len(self.anchor):] + Pathod.__init__(self, spec, application, request, **settings) + + +class PathodApp(tornado.web.Application): + LOGBUF = 500 + def __init__(self, **settings): + self.appsettings = settings + tornado.web.Application.__init__( + self, + [ + (r"/", Index), + (r"/log", Log), + (r"/log/clear", ClearLog), + (r"/log/([0-9]+)", OneLog), + (r"/help", Help), + (r"/preview", Preview), + (r"/api/shutdown", APIShutdown), + (r"/api/info", APIInfo), + (r"/api/log", APILog), + (r"/api/log/clear", APILogClear), + (r"/p/.*", RequestPathod, settings), + ], + static_path = utils.data.path("static"), + template_path = utils.data.path("templates"), + debug=True + ) + self.log = [] + self.logid = 0 + + def add_anchor(self, pattern, spec): + """ + Anchors are added to the beginning of the handlers. + """ + # We assume we have only one host... + l = self.handlers[0][1] + class FixedPathod(Pathod): + def __init__(self, application, request, **settings): + Pathod.__init__(self, spec, application, request, **settings) + FixedPathod.spec = spec + FixedPathod.pattern = pattern + l.insert(0, tornado.web.URLSpec(pattern, FixedPathod, self.appsettings)) + + def get_anchors(self): + """ + Anchors are added to the beginning of the handlers. + """ + l = self.handlers[0][1] + a = [] + for i in l: + if i.handler_class.__name__ == "FixedPathod": + a.append( + ( + i.handler_class.pattern, + i.handler_class.spec + ) + ) + return a + + def remove_anchor(self, pattern, spec): + """ + Anchors are added to the beginning of the handlers. + """ + l = self.handlers[0][1] + for i, h in enumerate(l): + if h.handler_class.__name__ == "FixedPathod": + if (h.handler_class.pattern, h.handler_class.spec) == (pattern, spec): + del l[i] + return + + def add_log(self, d): + d["id"] = self.logid + self.log.insert(0, d) + if len(self.log) > self.LOGBUF: + self.log.pop() + self.logid += 1 + + def log_by_id(self, id): + for i in self.log: + if i["id"] == id: + return i + + def clear_log(self): + self.log = [] + + def get_log(self): + return self.log + + +def make_app(staticdir=None, anchors=()): + """ + staticdir: A directory for static assets referenced in response patterns. + anchors: A sequence of strings of the form "pattern=pagespec" + """ + settings = dict( + staticdir=staticdir + ) + application = PathodApp(**settings) + for i in anchors: + rex, spec = utils.parse_anchor_spec(i, settings) + application.add_anchor(rex, spec) + return application + + +def make_server(application, port, address, ssl_options): + """ + Returns a (server, port) tuple. + + The returned port will match the passed port, unless the passed port + was 0. In that case, an arbitrary empty port will be bound to, and this + new port will be returned. + """ + http_server = tornado.httpserver.HTTPServer( + application, + ssl_options=ssl_options + ) + http_server.listen(port, address) + port = port + for i in http_server._sockets.values(): + sn = i.getsockname() + if sn[0] == address: + port = sn[1] + return http_server, port + + +# begin nocover +def run(server): + tornado.ioloop.IOLoop.instance().start() + server.stop() + + +class APILog(tornado.web.RequestHandler): + def get(self): + self.write( + dict( + d = self.application.get_log() + ) + ) + + +class APILogClear(tornado.web.RequestHandler): + def post(self): + self.application.clear_log() + self.write("OK") + + +class APIShutdown(tornado.web.RequestHandler): + def post(self): + tornado.ioloop.IOLoop.instance().stop() + self.write("OK") + + +class APIInfo(tornado.web.RequestHandler): + def get(self): + self.write( + dict( + version = version.IVERSION + ) + ) + + +class _Page(tornado.web.RequestHandler): + def render(self, name, **kwargs): + tornado.web.RequestHandler.render(self, name + ".html", **kwargs) + + +class Index(_Page): + name = "index" + section = "main" + def get(self): + self.render(self.name, section=self.section, spec="") + + +class Preview(_Page): + name = "preview" + section = "main" + SANITY = 1024*1024 + def get(self): + spec = self.get_argument("spec", None) + args = dict( + spec = spec, + section = self.section, + syntaxerror = None, + error = None + ) + try: + r = rparse.parse(self.application.settings, spec) + except rparse.ParseException, v: + args["syntaxerror"] = str(v) + args["marked"] = v.marked() + return self.render(self.name, **args) + if r.length() > self.SANITY: + error = "Refusing to preview a response of %s bytes. This is for your own good."%r.length() + args["error"] = error + else: + d = utils.DummyRequest() + r.serve(d) + args["output"] = d.getvalue() + self.render(self.name, **args) + + +class Help(_Page): + name = "help" + section = "help" + def get(self): + self.render(self.name, section=self.section) + + +class Log(_Page): + name = "log" + section = "log" + def get(self): + self.render(self.name, section=self.section, log=self.application.log) + + +class OneLog(_Page): + name = "onelog" + section = "log" + def get(self, lid): + l = pprint.pformat(self.application.log_by_id(int(lid))) + self.render(self.name, section=self.section, alog=l, lid=lid) + + +class ClearLog(_Page): + def post(self): + self.application.clear_logs() + self.redirect("/log") diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 476d26766..f712582e8 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -1,265 +1,14 @@ -import urllib, pprint -import tornado.web, tornado.template, tornado.ioloop, tornado.httpserver -import rparse, utils, version +import netlib + +class PathodHandler(netlib.BaseHandler): + def handle(self): + print "Here" -class Pathod(object): - def __init__(self, spec, application, request, **settings): - self.application, self.request, self.settings = application, request, settings - try: - self.response = rparse.parse(self.settings, spec) - except rparse.ParseException, v: - self.response = rparse.InternalResponse( - 800, - "Error parsing response spec: %s\n"%v.msg + v.marked() - ) - - def _execute(self, transforms, *args, **kwargs): - d = self.response.serve(self.request) - d["request"] = dict( - path = self.request.path, - method = self.request.method, - headers = self.request.headers, - host = self.request.host, - protocol = self.request.protocol, - remote_address = self.request.connection.address, - full_url = self.request.full_url(), - query = self.request.query, - version = self.request.version, - uri = self.request.uri, - ) - self.application.add_log(d) - - -class RequestPathod(Pathod): - anchor = "/p/" - def __init__(self, application, request, **settings): - spec = urllib.unquote(request.uri)[len(self.anchor):] - Pathod.__init__(self, spec, application, request, **settings) - - -class PathodApp(tornado.web.Application): - LOGBUF = 500 - def __init__(self, **settings): - self.appsettings = settings - tornado.web.Application.__init__( - self, - [ - (r"/", Index), - (r"/log", Log), - (r"/log/clear", ClearLog), - (r"/log/([0-9]+)", OneLog), - (r"/help", Help), - (r"/preview", Preview), - (r"/api/shutdown", APIShutdown), - (r"/api/info", APIInfo), - (r"/api/log", APILog), - (r"/api/log/clear", APILogClear), - (r"/p/.*", RequestPathod, settings), - ], - static_path = utils.data.path("static"), - template_path = utils.data.path("templates"), - debug=True - ) - self.log = [] - self.logid = 0 - - def add_anchor(self, pattern, spec): - """ - Anchors are added to the beginning of the handlers. - """ - # We assume we have only one host... - l = self.handlers[0][1] - class FixedPathod(Pathod): - def __init__(self, application, request, **settings): - Pathod.__init__(self, spec, application, request, **settings) - FixedPathod.spec = spec - FixedPathod.pattern = pattern - l.insert(0, tornado.web.URLSpec(pattern, FixedPathod, self.appsettings)) - - def get_anchors(self): - """ - Anchors are added to the beginning of the handlers. - """ - l = self.handlers[0][1] - a = [] - for i in l: - if i.handler_class.__name__ == "FixedPathod": - a.append( - ( - i.handler_class.pattern, - i.handler_class.spec - ) - ) - return a - - def remove_anchor(self, pattern, spec): - """ - Anchors are added to the beginning of the handlers. - """ - l = self.handlers[0][1] - for i, h in enumerate(l): - if h.handler_class.__name__ == "FixedPathod": - if (h.handler_class.pattern, h.handler_class.spec) == (pattern, spec): - del l[i] - return - - def add_log(self, d): - d["id"] = self.logid - self.log.insert(0, d) - if len(self.log) > self.LOGBUF: - self.log.pop() - self.logid += 1 - - def log_by_id(self, id): - for i in self.log: - if i["id"] == id: - return i - - def clear_log(self): - self.log = [] - - def get_log(self): - return self.log - - -def make_app(staticdir=None, anchors=()): - """ - staticdir: A directory for static assets referenced in response patterns. - anchors: A sequence of strings of the form "pattern=pagespec" - """ - settings = dict( - staticdir=staticdir - ) - application = PathodApp(**settings) - for i in anchors: - rex, spec = utils.parse_anchor_spec(i, settings) - application.add_anchor(rex, spec) - return application - - -def make_server(application, port, address, ssl_options): - """ - Returns a (server, port) tuple. - - The returned port will match the passed port, unless the passed port - was 0. In that case, an arbitrary empty port will be bound to, and this - new port will be returned. - """ - http_server = tornado.httpserver.HTTPServer( - application, - ssl_options=ssl_options - ) - http_server.listen(port, address) - port = port - for i in http_server._sockets.values(): - sn = i.getsockname() - if sn[0] == address: - port = sn[1] - return http_server, port - - -# begin nocover -def run(server): - tornado.ioloop.IOLoop.instance().start() - server.stop() - - -class APILog(tornado.web.RequestHandler): - def get(self): - self.write( - dict( - d = self.application.get_log() - ) - ) - - -class APILogClear(tornado.web.RequestHandler): - def post(self): - self.application.clear_log() - self.write("OK") - - -class APIShutdown(tornado.web.RequestHandler): - def post(self): - tornado.ioloop.IOLoop.instance().stop() - self.write("OK") - - -class APIInfo(tornado.web.RequestHandler): - def get(self): - self.write( - dict( - version = version.IVERSION - ) - ) - - -class _Page(tornado.web.RequestHandler): - def render(self, name, **kwargs): - tornado.web.RequestHandler.render(self, name + ".html", **kwargs) - - -class Index(_Page): - name = "index" - section = "main" - def get(self): - self.render(self.name, section=self.section, spec="") - - -class Preview(_Page): - name = "preview" - section = "main" - SANITY = 1024*1024 - def get(self): - spec = self.get_argument("spec", None) - args = dict( - spec = spec, - section = self.section, - syntaxerror = None, - error = None - ) - try: - r = rparse.parse(self.application.settings, spec) - except rparse.ParseException, v: - args["syntaxerror"] = str(v) - args["marked"] = v.marked() - return self.render(self.name, **args) - if r.length() > self.SANITY: - error = "Refusing to preview a response of %s bytes. This is for your own good."%r.length() - args["error"] = error - else: - d = utils.DummyRequest() - r.serve(d) - args["output"] = d.getvalue() - self.render(self.name, **args) - - -class Help(_Page): - name = "help" - section = "help" - def get(self): - self.render(self.name, section=self.section) - - -class Log(_Page): - name = "log" - section = "log" - def get(self): - self.render(self.name, section=self.section, log=self.application.log) - - -class OneLog(_Page): - name = "onelog" - section = "log" - def get(self, lid): - l = pprint.pformat(self.application.log_by_id(int(lid))) - self.render(self.name, section=self.section, alog=l, lid=lid) - - -class ClearLog(_Page): - def post(self): - self.application.clear_logs() - self.redirect("/log") +class PathodServer(netlib.TCPServer): + def __init__(self, addr): + netlib.TCPServer.__init__(self, addr) + def handle_connection(self, request, client_address): + PathodHandler(request, client_address, self) diff --git a/libpathod/test.py b/libpathod/test.py index dcbb65d51..a66124a33 100644 --- a/libpathod/test.py +++ b/libpathod/test.py @@ -29,7 +29,6 @@ class PaThread(threading.Thread): self.q, self.app, self.ssl = q, app, ssl self.port = None -# begin nocover def run(self): if self.ssl is True: ssloptions = dict( diff --git a/opathod b/opathod new file mode 100755 index 000000000..f49baa347 --- /dev/null +++ b/opathod @@ -0,0 +1,56 @@ +#!/usr/bin/env python +import argparse, sys +from libpathod import pathod, utils, version +import tornado.ioloop + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument("-p", dest='port', default=9999, type=int, help='Port. Specify 0 to pick an arbitrary empty port.') + parser.add_argument("-l", dest='address', default="0.0.0.0", type=str, help='Listening address.') + parser.add_argument( + "-a", dest='anchors', default=[], type=str, action="append", + help='Add an anchor. Specified as a string with the form pattern=pagespec' + ) + parser.add_argument( + "-d", dest='staticdir', default=None, type=str, + help='Directory for static files.' + ) + parser.add_argument( + "-s", dest='ssl', default=False, + action="store_true", + help='Serve with SSL.' + ) + parser.add_argument( + "--keyfile", dest='ssl_keyfile', default=None, + type=str, + help='SSL key file. If not specified, a default key is used.' + ) + parser.add_argument( + "--certfile", dest='ssl_certfile', default=None, + type=str, + help='SSL cert file. If not specified, a default cert is used.' + ) + args = parser.parse_args() + + try: + app = pathod.make_app(staticdir=args.staticdir, anchors=args.anchors) + except utils.AnchorError, v: + parser.error(str(v)) + + sl = [args.ssl_keyfile, args.ssl_certfile] + if any(sl) and not all(sl): + parser.error("Both --certfile and --keyfile must be specified.") + + if args.ssl: + ssl = dict( + keyfile = args.ssl_keyfile or utils.data.path("resources/server.key"), + certfile = args.ssl_certfile or utils.data.path("resources/server.crt"), + ) + else: + ssl = None + try: + server, port = pathod.make_server(app, args.port, args.address, ssl) + print "%s listening on port %s"%(version.NAMEVERSION, port) + pathod.run(server) + except KeyboardInterrupt: + pass diff --git a/pathod b/pathod index f49baa347..910b936d9 100755 --- a/pathod +++ b/pathod @@ -1,56 +1,5 @@ #!/usr/bin/env python -import argparse, sys -from libpathod import pathod, utils, version -import tornado.ioloop +from libpathod import pathod -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Process some integers.') - parser.add_argument("-p", dest='port', default=9999, type=int, help='Port. Specify 0 to pick an arbitrary empty port.') - parser.add_argument("-l", dest='address', default="0.0.0.0", type=str, help='Listening address.') - parser.add_argument( - "-a", dest='anchors', default=[], type=str, action="append", - help='Add an anchor. Specified as a string with the form pattern=pagespec' - ) - parser.add_argument( - "-d", dest='staticdir', default=None, type=str, - help='Directory for static files.' - ) - parser.add_argument( - "-s", dest='ssl', default=False, - action="store_true", - help='Serve with SSL.' - ) - parser.add_argument( - "--keyfile", dest='ssl_keyfile', default=None, - type=str, - help='SSL key file. If not specified, a default key is used.' - ) - parser.add_argument( - "--certfile", dest='ssl_certfile', default=None, - type=str, - help='SSL cert file. If not specified, a default cert is used.' - ) - args = parser.parse_args() - - try: - app = pathod.make_app(staticdir=args.staticdir, anchors=args.anchors) - except utils.AnchorError, v: - parser.error(str(v)) - - sl = [args.ssl_keyfile, args.ssl_certfile] - if any(sl) and not all(sl): - parser.error("Both --certfile and --keyfile must be specified.") - - if args.ssl: - ssl = dict( - keyfile = args.ssl_keyfile or utils.data.path("resources/server.key"), - certfile = args.ssl_certfile or utils.data.path("resources/server.crt"), - ) - else: - ssl = None - try: - server, port = pathod.make_server(app, args.port, args.address, ssl) - print "%s listening on port %s"%(version.NAMEVERSION, port) - pathod.run(server) - except KeyboardInterrupt: - pass +s = pathod.PathodServer(("127.0.0.1", 8888)) +s.serve_forever()