Factor logger out of pathoc, use it in pathod as well.

libpathod/log.py

@@ -0,0 +1,57 @@
+import netlib.utils
+import netlib.tcp
+import netlib.http
+
+
+class Log:
+    def __init__(self, fp, hex, rfile, wfile):
+        self.lines = []
+        self.fp = fp
+        self.suppressed = False
+        self.hex = hex
+        self.rfile, self.wfile = rfile, wfile
+
+    def __enter__(self):
+        if self.wfile:
+            self.wfile.start_log()
+        if self.rfile:
+            self.rfile.start_log()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        wlog = self.wfile.get_log() if self.wfile else None
+        rlog = self.rfile.get_log() if self.rfile else None
+        if self.suppressed or not self.fp:
+            return
+        if wlog:
+            self("Bytes written:")
+            self.dump(wlog, self.hex)
+        if rlog:
+            self("Bytes read:")
+            self.dump(rlog, self.hex)
+        if exc_type == netlib.tcp.NetLibTimeout:
+            self("Timeout")
+        elif exc_type in (
+            netlib.tcp.NetLibDisconnect,
+            netlib.http.HttpErrorConnClosed
+        ):
+            self("Disconnected")
+        elif exc_type == netlib.http.HttpError:
+            self("HTTP Error: %s" % exc_value.message)
+        self.fp.write("\n".join(self.lines))
+        self.fp.write("\n")
+        self.fp.flush()
+
+    def suppress(self):
+        self.suppressed = True
+
+    def dump(self, data, hexdump):
+        if hexdump:
+            for line in netlib.utils.hexdump(data):
+                self("\t%s %s %s" % line)
+        else:
+            for i in netlib.utils.cleanBin(data).split("\n"):
+                self("\t%s" % i)
+
+    def __call__(self, line):
+        self.lines.append(line)

libpathod/pathoc.py

@@ -10,68 +10,16 @@ import threading
 import OpenSSL.crypto
 
 from netlib import tcp, http, certutils, websockets
-import netlib.utils
 
 import language.http
 import language.websockets
-import utils
+from . import utils, log
 
 
 class PathocError(Exception):
     pass
 
 
-class Log:
-    def __init__(self, fp, hex, rfile, wfile):
-        self.lines = []
-        self.fp = fp
-        self.suppressed = False
-        self.hex = hex
-        self.rfile, self.wfile = rfile, wfile
-
-    def __enter__(self):
-        if self.wfile:
-            self.wfile.start_log()
-        if self.rfile:
-            self.rfile.start_log()
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        wlog = self.wfile.get_log() if self.wfile else None
-        rlog = self.rfile.get_log() if self.rfile else None
-        if self.suppressed or not self.fp:
-            return
-        if wlog:
-            self("Bytes written:")
-            self.dump(wlog, self.hex)
-        if rlog:
-            self("Bytes read:")
-            self.dump(rlog, self.hex)
-        if exc_type == tcp.NetLibTimeout:
-            self("Timeout")
-        elif exc_type in (tcp.NetLibDisconnect, http.HttpErrorConnClosed):
-            self("Disconnected")
-        elif exc_type == http.HttpError:
-            self("HTTP Error: %s" % exc_value.message)
-        self.fp.write("\n".join(self.lines))
-        self.fp.write("\n")
-        self.fp.flush()
-
-    def suppress(self):
-        self.suppressed = True
-
-    def dump(self, data, hexdump):
-        if hexdump:
-            for line in netlib.utils.hexdump(data):
-                self("\t%s %s %s" % line)
-        else:
-            for i in netlib.utils.cleanBin(data).split("\n"):
-                self("\t%s" % i)
-
-    def __call__(self, line):
-        self.lines.append(line)
-
-
 class SSLInfo:
     def __init__(self, certchain, cipher):
         self.certchain, self.cipher = certchain, cipher
@@ -149,7 +97,7 @@ class WebsocketFrameReader(threading.Thread):
         self.is_done = Queue.Queue()
 
     def log(self, rfile):
-        return Log(
+        return log.Log(
             self.logfp,
             self.hexdump,
             rfile if self.showresp else None,
@@ -242,7 +190,7 @@ class Pathoc(tcp.TCPClient):
         self.ws_framereader = None
 
     def log(self):
-        return Log(
+        return log.Log(
             self.fp,
             self.hexdump,
             self.rfile if self.showresp else None,

libpathod/pathod.py

@@ -7,7 +7,7 @@ import urllib
 from netlib import tcp, http, wsgi, certutils, websockets
 import netlib.utils
 
-from . import version, app, language, utils
+from . import version, app, language, utils, log
 import language.http
 import language.actions
 
@@ -60,7 +60,8 @@ class PathodHandler(tcp.BaseHandler):
     wbufsize = 0
     sni = None
 
-    def __init__(self, connection, address, server, settings):
+    def __init__(self, connection, address, server, logfp, settings):
+        self.logfp = logfp
         tcp.BaseHandler.__init__(self, connection, address, server)
         self.settings = copy.copy(settings)
 
@@ -106,11 +107,9 @@ class PathodHandler(tcp.BaseHandler):
             again: True if request handling should continue.
             log: A dictionary, or None
         """
-        if self.server.logreq:
+        lr = self.rfile if self.server.logreq else None
-            self.rfile.start_log()
+        lw = self.wfile if self.server.logresp else None
-        if self.server.logresp:
+        with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg:
-            self.wfile.start_log()
-
             line = http.get_request_line(self.rfile)
             if not line:
                 # Normal termination
@@ -140,7 +139,7 @@ class PathodHandler(tcp.BaseHandler):
                         )
                     except tcp.NetLibError as v:
                         s = str(v)
-                    self.info(s)
+                        lg(s)
                         self.addlog(dict(type="error", msg=s))
                         return False
                 return True
@@ -150,14 +149,14 @@ class PathodHandler(tcp.BaseHandler):
                 method, path, httpversion = m.v
             else:
                 s = "Invalid first line: %s" % repr(line)
-            self.info(s)
+                lg(s)
                 self.addlog(dict(type="error", msg=s))
                 return False
 
             headers = http.read_headers(self.rfile)
             if headers is None:
                 s = "Invalid headers"
-            self.info(s)
+                lg(s)
                 self.addlog(dict(type="error", msg=s))
                 return False
 
@@ -195,13 +194,13 @@ class PathodHandler(tcp.BaseHandler):
                 )
             except http.HttpError as s:
                 s = str(s)
-            self.info(s)
+                lg(s)
                 self.addlog(dict(type="error", msg=s))
                 return False
 
             for i in self.server.anchors:
                 if i[0].match(path):
-                self.info("crafting anchor: %s" % path)
+                    lg("crafting anchor: %s" % path)
                     again, retlog["response"] = self.serve_crafted(i[1])
                     self.addlog(retlog)
                     return again
@@ -214,11 +213,11 @@ class PathodHandler(tcp.BaseHandler):
                 self.settings.websocket_key = key
                 if key and not spec:
                     spec = "ws"
-            self.info("crafting spec: %s" % spec)
+                lg("crafting spec: %s" % spec)
                 try:
                     crafted = language.parse_response(spec)
                 except language.ParseException as v:
-                self.info("Parse error: %s" % v.msg)
+                    lg("Parse error: %s" % v.msg)
                     crafted = language.http.make_error_response(
                         "Parse Error",
                         "Error parsing response spec: %s\n" % v.msg + v.marked()
@@ -235,7 +234,7 @@ class PathodHandler(tcp.BaseHandler):
                 ))
                 return False
             else:
-            self.info("app: %s %s" % (method, path))
+                lg("app: %s %s" % (method, path))
                 req = wsgi.Request("http", method, path, headers, content)
                 flow = wsgi.Flow(self.address, req)
                 sn = self.connection.getsockname()
@@ -248,28 +247,15 @@ class PathodHandler(tcp.BaseHandler):
                 a.serve(flow, self.wfile)
                 return True
 
-    def _log_bytes(self, header, data, hexdump):
-        s = []
-        if hexdump:
-            s.append("%s (hex dump):" % header)
-            for line in netlib.utils.hexdump(data):
-                s.append("\t%s %s %s" % line)
-        else:
-            s.append("%s (unprintables escaped):" % header)
-            s.append(netlib.utils.cleanBin(data))
-        self.info("\n".join(s))
-
     def addlog(self, log):
         # FIXME: The bytes in the log should not be escaped. We do this at the
         # moment because JSON encoding can't handle binary data, and I don't
         # want to base64 everything.
         if self.server.logreq:
             bytes = self.rfile.get_log().encode("string_escape")
-            self._log_bytes("Request", bytes, self.server.hexdump)
             log["request_bytes"] = bytes
         if self.server.logresp:
             bytes = self.wfile.get_log().encode("string_escape")
-            self._log_bytes("Response", bytes, self.server.hexdump)
             log["response_bytes"] = bytes
         self.server.add_log(log)
 
@@ -324,6 +310,7 @@ class Pathod(tcp.TCPServer):
         explain=False,
         hexdump=False,
         webdebug=False,
+        logfp=sys.stdout,
     ):
         """
             addr: (address, port) tuple. If port is 0, a free port will be
@@ -350,6 +337,7 @@ class Pathod(tcp.TCPServer):
         self.timeout, self.logreq = timeout, logreq
         self.logresp, self.hexdump = logresp, hexdump
         self.explain = explain
+        self.logfp = logfp
 
         self.app = app.make_app(noapi, webdebug)
         self.app.config["pathod"] = self
@@ -378,7 +366,13 @@ class Pathod(tcp.TCPServer):
         return None, req
 
     def handle_client_connection(self, request, client_address):
-        h = PathodHandler(request, client_address, self, self.settings)
+        h = PathodHandler(
+            request,
+            client_address,
+            self,
+            self.logfp,
+            self.settings
+        )
         try:
             h.handle()
             h.finish()

libpathod/test.py

@@ -83,6 +83,7 @@ class _PaThread(threading.Thread):
         self.server = pathod.Pathod(
             (self.iface, 0),
             ssl = self.ssl,
+            logfp = None,
             **self.daemonargs
         )
         self.name = "PathodThread (%s:%s)" % (

libpathod/utils.py

@@ -10,11 +10,11 @@ SSLVERSIONS = {
 }
 
 SIZE_UNITS = dict(
-    b = 1024**0,
+    b = 1024 ** 0,
-    k = 1024**1,
+    k = 1024 ** 1,
-    m = 1024**2,
+    m = 1024 ** 2,
-    g = 1024**3,
+    g = 1024 ** 3,
-    t = 1024**4,
+    t = 1024 ** 4,
 )
 
 

test/test_language_base.py

@@ -85,7 +85,7 @@ class TestTokValueGenerate:
         v = base.TokValue.parseString("@10k")[0]
         assert v.bytes() == 10240
         v = base.TokValue.parseString("@10g")[0]
-        assert v.bytes() == 1024**3 * 10
+        assert v.bytes() == 1024 ** 3 * 10
 
         v = base.TokValue.parseString("@10g,digits")[0]
         assert v.datatype == "digits"

test/test_pathod.py

@@ -1,3 +1,4 @@
+import cStringIO
 from libpathod import pathod, version
 from netlib import tcp, http
 import tutils
@@ -5,7 +6,8 @@ import tutils
 
 class TestPathod(object):
     def test_logging(self):
-        p = pathod.Pathod(("127.0.0.1", 0))
+        s = cStringIO.StringIO()
+        p = pathod.Pathod(("127.0.0.1", 0), logfp=s)
         assert len(p.get_log()) == 0
         id = p.add_log(dict(s="foo"))
         assert p.log_by_id(id)

test/test_utils.py

@@ -20,7 +20,7 @@ def test_parse_size():
 
 def test_parse_anchor_spec():
     assert utils.parse_anchor_spec("foo=200") == ("foo", "200")
-    assert utils.parse_anchor_spec("foo") == None
+    assert utils.parse_anchor_spec("foo") is None
 
 
 def test_data_path():