Massive refactoring to split up language implementation.

libpathod/language/__init__.py

@@ -0,0 +1,88 @@
+import time
+
+import contrib.pyparsing as pp
+
+from . import base, http, websockets, writer, exceptions
+
+from exceptions import *
+
+
+class Settings:
+    def __init__(
+        self,
+        staticdir = None,
+        unconstrained_file_access = False,
+        request_host = None,
+        websocket_key = None
+    ):
+        self.staticdir = staticdir
+        self.unconstrained_file_access = unconstrained_file_access
+        self.request_host = request_host
+        self.websocket_key = websocket_key
+
+
+def parse_response(s):
+    """
+        May raise ParseException
+    """
+    try:
+        s = s.decode("ascii")
+    except UnicodeError:
+        raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
+    try:
+        return http.Response.expr().parseString(s, parseAll=True)[0]
+    except pp.ParseException, v:
+        raise exceptions.ParseException(v.msg, v.line, v.col)
+
+
+def parse_requests(s):
+    """
+        May raise ParseException
+    """
+    try:
+        s = s.decode("ascii")
+    except UnicodeError:
+        raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
+    try:
+        return pp.OneOrMore(
+            pp.Or(
+                [
+                    websockets.WebsocketFrame.expr(),
+                    http.Request.expr(),
+                ]
+            )
+        ).parseString(s, parseAll=True)
+    except pp.ParseException, v:
+        raise exceptions.ParseException(v.msg, v.line, v.col)
+
+
+def serve(msg, fp, settings):
+    """
+        fp: The file pointer to write to.
+
+        request_host: If this a request, this is the connecting host. If
+        None, we assume it's a response. Used to decide what standard
+        modifications to make if raw is not set.
+
+        Calling this function may modify the object.
+    """
+    msg = msg.resolve(settings)
+    started = time.time()
+
+    vals = msg.values(settings)
+    vals.reverse()
+
+    actions = msg.actions[:]
+    actions.sort()
+    actions.reverse()
+    actions = [i.intermediate(settings) for i in actions]
+
+    disconnect = writer.write_values(fp, vals, actions[:])
+    duration = time.time() - started
+    ret = dict(
+        disconnect = disconnect,
+        started = started,
+        duration = duration,
+    )
+    ret.update(msg.log(settings))
+    return ret

libpathod/language/base.py

@@ -1,34 +1,17 @@
 import operator
-import string
 import random
-import mmap
 import os
-import time
 import copy
 import abc
 import contrib.pyparsing as pp
-from netlib import http_status, tcp, http_uastrings, websockets
+from netlib import http_uastrings
 
-import utils
+from .. import utils
+from . import generators, exceptions
 
-BLOCKSIZE = 1024
 TRUNCATE = 1024
 
 
-class Settings:
-    def __init__(
-        self,
-        staticdir = None,
-        unconstrained_file_access = False,
-        request_host = None,
-        websocket_key = None
-    ):
-        self.staticdir = staticdir
-        self.unconstrained_file_access = unconstrained_file_access
-        self.request_host = request_host
-        self.websocket_key = websocket_key
-
-
 def quote(s):
     quotechar = s[0]
     s = s[1:-1]
@@ -36,131 +19,6 @@ def quote(s):
     return quotechar + s + quotechar
 
 
-class RenderError(Exception):
-    pass
-
-
-class FileAccessDenied(RenderError):
-    pass
-
-
-class ParseException(Exception):
-    def __init__(self, msg, s, col):
-        Exception.__init__(self)
-        self.msg = msg
-        self.s = s
-        self.col = col
-
-    def marked(self):
-        return "%s\n%s"%(self.s, " " * (self.col - 1) + "^")
-
-    def __str__(self):
-        return "%s at char %s"%(self.msg, self.col)
-
-
-def send_chunk(fp, val, blocksize, start, end):
-    """
-        (start, end): Inclusive lower bound, exclusive upper bound.
-    """
-    for i in range(start, end, blocksize):
-        fp.write(
-            val[i:min(i + blocksize, end)]
-        )
-    return end - start
-
-
-def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE):
-    """
-        vals: A list of values, which may be strings or Value objects.
-
-        actions: A list of (offset, action, arg) tuples. Action may be "pause"
-        or "disconnect".
-
-        Both vals and actions are in reverse order, with the first items last.
-
-        Return True if connection should disconnect.
-    """
-    sofar = 0
-    try:
-        while vals:
-            v = vals.pop()
-            offset = 0
-            while actions and actions[-1][0] < (sofar + len(v)):
-                a = actions.pop()
-                offset += send_chunk(
-                    fp,
-                    v,
-                    blocksize,
-                    offset,
-                    a[0] - sofar - offset
-                )
-                if a[1] == "pause":
-                    time.sleep(a[2])
-                elif a[1] == "disconnect":
-                    return True
-                elif a[1] == "inject":
-                    send_chunk(fp, a[2], blocksize, 0, len(a[2]))
-            send_chunk(fp, v, blocksize, offset, len(v))
-            sofar += len(v)
-        # Remainders
-        while actions:
-            a = actions.pop()
-            if a[1] == "pause":
-                time.sleep(a[2])
-            elif a[1] == "disconnect":
-                return True
-            elif a[1] == "inject":
-                send_chunk(fp, a[2], blocksize, 0, len(a[2]))
-    except tcp.NetLibDisconnect: # pragma: no cover
-        return True
-
-
-def serve(msg, fp, settings):
-    """
-        fp: The file pointer to write to.
-
-        request_host: If this a request, this is the connecting host. If
-        None, we assume it's a response. Used to decide what standard
-        modifications to make if raw is not set.
-
-        Calling this function may modify the object.
-    """
-    msg = msg.resolve(settings)
-    started = time.time()
-
-    vals = msg.values(settings)
-    vals.reverse()
-
-    actions = msg.actions[:]
-    actions.sort()
-    actions.reverse()
-    actions = [i.intermediate(settings) for i in actions]
-
-    disconnect = write_values(fp, vals, actions[:])
-    duration = time.time() - started
-    ret = dict(
-        disconnect = disconnect,
-        started = started,
-        duration = duration,
-    )
-    ret.update(msg.log(settings))
-    return ret
-
-
-DATATYPES = dict(
-    ascii_letters = string.ascii_letters,
-    ascii_lowercase = string.ascii_lowercase,
-    ascii_uppercase = string.ascii_uppercase,
-    digits = string.digits,
-    hexdigits = string.hexdigits,
-    octdigits = string.octdigits,
-    punctuation = string.punctuation,
-    whitespace = string.whitespace,
-    ascii = string.printable,
-    bytes = "".join(chr(i) for i in range(256))
-)
-
-
 v_integer = pp.Word(pp.nums)\
     .setName("integer")\
     .setParseAction(lambda toks: int(toks[0]))
@@ -191,89 +49,6 @@ v_naked_literal = pp.MatchFirst(
 )
 
 
-class TransformGenerator:
-    """
-        Perform a byte-by-byte transform another generator - that is, for each
-        input byte, the transformation must produce one output byte.
-
-        gen: A generator to wrap
-        transform: A function (offset, data) -> transformed
-    """
-    def __init__(self, gen, transform):
-        self.gen = gen
-        self.transform = transform
-
-    def __len__(self):
-        return len(self.gen)
-
-    def __getitem__(self, x):
-        d = self.gen.__getitem__(x)
-        return self.transform(x, d)
-
-    def __getslice__(self, a, b):
-        d = self.gen.__getslice__(a, b)
-        return self.transform(a, d)
-
-    def __repr__(self):
-        return "'%s'"%self.gen
-
-
-class LiteralGenerator:
-    def __init__(self, s):
-        self.s = s
-
-    def __len__(self):
-        return len(self.s)
-
-    def __getitem__(self, x):
-        return self.s.__getitem__(x)
-
-    def __getslice__(self, a, b):
-        return self.s.__getslice__(a, b)
-
-    def __repr__(self):
-        return "'%s'"%self.s
-
-
-class RandomGenerator:
-    def __init__(self, dtype, length):
-        self.dtype = dtype
-        self.length = length
-
-    def __len__(self):
-        return self.length
-
-    def __getitem__(self, x):
-        return random.choice(DATATYPES[self.dtype])
-
-    def __getslice__(self, a, b):
-        b = min(b, self.length)
-        chars = DATATYPES[self.dtype]
-        return "".join(random.choice(chars) for x in range(a, b))
-
-    def __repr__(self):
-        return "%s random from %s"%(self.length, self.dtype)
-
-
-class FileGenerator:
-    def __init__(self, path):
-        self.path = path
-        self.fp = file(path, "rb")
-        self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ)
-
-    def __len__(self):
-        return len(self.map)
-
-    def __getitem__(self, x):
-        return self.map.__getitem__(x)
-
-    def __getslice__(self, a, b):
-        return self.map.__getslice__(a, b)
-
-    def __repr__(self):
-        return "<%s"%self.path
-
-
 class _Token(object):
     """
         A specification token. Tokens are immutable.
@@ -310,7 +85,7 @@ class _ValueLiteral(_Token):
         self.val = val.decode("string_escape")
 
     def get_generator(self, settings):
-        return LiteralGenerator(self.val)
+        return generators.LiteralGenerator(self.val)
 
     def freeze(self, settings):
         return self
@@ -352,7 +127,7 @@ class ValueGenerate(_Token):
         return self.usize * utils.SIZE_UNITS[self.unit]
 
     def get_generator(self, settings):
-        return RandomGenerator(self.datatype, self.bytes())
+        return generators.RandomGenerator(self.datatype, self.bytes())
 
     def freeze(self, settings):
         g = self.get_generator(settings)
@@ -369,7 +144,10 @@ class ValueGenerate(_Token):
         e = e + pp.Optional(u, default=None)
 
         s = pp.Literal(",").suppress()
-        s += reduce(operator.or_, [pp.Literal(i) for i in DATATYPES.keys()])
+        s += reduce(
+            operator.or_,
+            [pp.Literal(i) for i in generators.DATATYPES.keys()]
+        )
         e += pp.Optional(s, default="bytes")
         return e.setParseAction(lambda x: klass(*x))
 
@@ -397,19 +175,19 @@ class ValueFile(_Token):
 
     def get_generator(self, settings):
         if not settings.staticdir:
-            raise FileAccessDenied("File access disabled.")
+            raise exceptions.FileAccessDenied("File access disabled.")
         s = os.path.expanduser(self.path)
         s = os.path.normpath(
             os.path.abspath(os.path.join(settings.staticdir, s))
         )
         uf = settings.unconstrained_file_access
         if not uf and not s.startswith(settings.staticdir):
-            raise FileAccessDenied(
+            raise exceptions.FileAccessDenied(
                 "File access outside of configured directory"
             )
         if not os.path.isfile(s):
-            raise FileAccessDenied("File not readable")
-        return FileGenerator(s)
+            raise exceptions.FileAccessDenied("File not readable")
+        return generators.FileGenerator(s)
 
     def spec(self):
         return "<'%s'"%self.path.encode("string_escape")
@@ -587,14 +365,15 @@ class PathodSpec(_Token):
     def __init__(self, value):
         self.value = value
         try:
-            self.parsed = Response(
-                Response.expr().parseString(
+            import http
+            self.parsed = http.Response(
+                http.Response.expr().parseString(
                     value.val,
                     parseAll=True
                 )
             )
         except pp.ParseException, v:
-            raise ParseException(v.msg, v.line, v.col)
+            raise exceptions.ParseException(v.msg, v.line, v.col)
 
     @classmethod
     def expr(klass):
@@ -719,7 +498,7 @@ class Code(_Component):
         return e.setParseAction(lambda x: klass(*x))
 
     def values(self, settings):
-        return [LiteralGenerator(self.code)]
+        return [generators.LiteralGenerator(self.code)]
 
     def spec(self):
         return "%s"%(self.code)
@@ -956,339 +735,17 @@ class _Message(object):
 Sep = pp.Optional(pp.Literal(":")).suppress()
 
 
-class _HTTPMessage(_Message):
-    version = "HTTP/1.1"
-    @abc.abstractmethod
-    def preamble(self, settings): # pragma: no cover
-        pass
-
-    def values(self, settings):
-        vals = self.preamble(settings)
-        vals.append("\r\n")
-        for h in self.headers:
-            vals.extend(h.values(settings))
-        vals.append("\r\n")
-        if self.body:
-            vals.append(self.body.value.get_generator(settings))
-        return vals
-
-
-class Response(_HTTPMessage):
-    comps = (
-        Body,
-        Header,
-        PauseAt,
-        DisconnectAt,
-        InjectAt,
-        ShortcutContentType,
-        ShortcutLocation,
-        Raw,
-        Reason
-    )
-    logattrs = ["code", "reason", "version", "body"]
-
-    @property
-    def ws(self):
-        return self.tok(WS)
-
-    @property
-    def code(self):
-        return self.tok(Code)
-
-    @property
-    def reason(self):
-        return self.tok(Reason)
-
-    def preamble(self, settings):
-        l = [self.version, " "]
-        l.extend(self.code.values(settings))
-        code = int(self.code.code)
-        l.append(" ")
-        if self.reason:
-            l.extend(self.reason.values(settings))
-        else:
-            l.append(
-                LiteralGenerator(
-                    http_status.RESPONSES.get(
-                        code,
-                        "Unknown code"
-                    )
-                )
-            )
-        return l
-
-    def resolve(self, settings, msg=None):
-        tokens = self.tokens[:]
-        if self.ws:
-            if not settings.websocket_key:
-                raise RenderError(
-                    "No websocket key - have we seen a client handshake?"
-                )
-            if not self.code:
-                tokens.insert(
-                    1,
-                    Code(101)
-                )
-            hdrs = websockets.server_handshake_headers(settings.websocket_key)
-            for i in hdrs.lst:
-                if not utils.get_header(i[0], self.headers):
-                    tokens.append(
-                        Header(ValueLiteral(i[0]), ValueLiteral(i[1]))
-                    )
-        if not self.raw:
-            if not utils.get_header("Content-Length", self.headers):
-                if not self.body:
-                    length = 0
-                else:
-                    length = len(self.body.value.get_generator(settings))
-                tokens.append(
-                    Header(
-                        ValueLiteral("Content-Length"),
-                        ValueLiteral(str(length)),
-                    )
-                )
-        intermediate = self.__class__(tokens)
-        return self.__class__(
-            [i.resolve(settings, intermediate) for i in tokens]
-        )
-
-    @classmethod
-    def expr(klass):
-        parts = [i.expr() for i in klass.comps]
-        atom = pp.MatchFirst(parts)
-        resp = pp.And(
-            [
-                pp.MatchFirst(
-                    [
-                        WS.expr() + pp.Optional(Sep + Code.expr()),
-                        Code.expr(),
-                    ]
-                ),
-                pp.ZeroOrMore(Sep + atom)
-            ]
-        )
-        resp = resp.setParseAction(klass)
-        return resp
-
-    def spec(self):
-        return ":".join([i.spec() for i in self.tokens])
-
-
-class Request(_HTTPMessage):
-    comps = (
-        Body,
-        Header,
-        PauseAt,
-        DisconnectAt,
-        InjectAt,
-        ShortcutContentType,
-        ShortcutUserAgent,
-        Raw,
-        PathodSpec,
-    )
-    logattrs = ["method", "path", "body"]
-
-    @property
-    def ws(self):
-        return self.tok(WS)
-
-    @property
-    def method(self):
-        return self.tok(Method)
-
-    @property
-    def path(self):
-        return self.tok(Path)
-
-    @property
-    def pathodspec(self):
-        return self.tok(PathodSpec)
-
-    def preamble(self, settings):
-        v = self.method.values(settings)
-        v.append(" ")
-        v.extend(self.path.values(settings))
-        if self.pathodspec:
-            v.append(self.pathodspec.parsed.spec())
-        v.append(" ")
-        v.append(self.version)
-        return v
-
-    def resolve(self, settings, msg=None):
-        tokens = self.tokens[:]
-        if self.ws:
-            if not self.method:
-                tokens.insert(
-                    1,
-                    Method("get")
-                )
-            for i in websockets.client_handshake_headers().lst:
-                if not utils.get_header(i[0], self.headers):
-                    tokens.append(
-                        Header(ValueLiteral(i[0]), ValueLiteral(i[1]))
-                    )
-        if not self.raw:
-            if not utils.get_header("Content-Length", self.headers):
-                if self.body:
-                    length = len(self.body.value.get_generator(settings))
-                    tokens.append(
-                        Header(
-                            ValueLiteral("Content-Length"),
-                            ValueLiteral(str(length)),
-                        )
-                    )
-            if settings.request_host:
-                if not utils.get_header("Host", self.headers):
-                    tokens.append(
-                        Header(
-                            ValueLiteral("Host"),
-                            ValueLiteral(settings.request_host)
-                        )
-                    )
-        intermediate = self.__class__(tokens)
-        return self.__class__(
-            [i.resolve(settings, intermediate) for i in tokens]
-        )
-
-    @classmethod
-    def expr(klass):
-        parts = [i.expr() for i in klass.comps]
-        atom = pp.MatchFirst(parts)
-        resp = pp.And(
-            [
-                pp.MatchFirst(
-                    [
-                        WS.expr() + pp.Optional(Sep + Method.expr()),
-                        Method.expr(),
-                    ]
-                ),
-                Sep,
-                Path.expr(),
-                pp.ZeroOrMore(Sep + atom)
-            ]
-        )
-        resp = resp.setParseAction(klass)
-        return resp
-
-    def spec(self):
-        return ":".join([i.spec() for i in self.tokens])
-
-
-class WebsocketFrame(_Message):
-    comps = (
-        Body,
-        PauseAt,
-        DisconnectAt,
-        InjectAt
-    )
-    logattrs = ["body"]
-
-    @classmethod
-    def expr(klass):
-        parts = [i.expr() for i in klass.comps]
-        atom = pp.MatchFirst(parts)
-        resp = pp.And(
-            [
-                WF.expr(),
-                Sep,
-                pp.ZeroOrMore(Sep + atom)
-            ]
-        )
-        resp = resp.setParseAction(klass)
-        return resp
-
-    def values(self, settings):
-        vals = []
-        if self.body:
-            bodygen = self.body.value.get_generator(settings)
-            length = len(self.body.value.get_generator(settings))
-        else:
-            bodygen = None
-            length = 0
-        frame = websockets.FrameHeader(
-            mask = True,
-            payload_length = length
-        )
-        vals = [frame.to_bytes()]
-        if self.body:
-            masker = websockets.Masker(frame.masking_key)
-            vals.append(
-                TransformGenerator(
-                    bodygen,
-                    masker.mask
-                )
-            )
-        return vals
-
-    def resolve(self, settings, msg=None):
-        return self.__class__(
-            [i.resolve(settings, msg) for i in self.tokens]
-        )
-
-    def spec(self):
-        return ":".join([i.spec() for i in self.tokens])
-
-
-class PathodErrorResponse(Response):
-    pass
-
-
-def make_error_response(reason, body=None):
-    tokens = [
-        Code("800"),
-        Header(ValueLiteral("Content-Type"), ValueLiteral("text/plain")),
-        Reason(ValueLiteral(reason)),
-        Body(ValueLiteral("pathod error: " + (body or reason))),
-    ]
-    return PathodErrorResponse(tokens)
-
-
 def read_file(settings, s):
     uf = settings.get("unconstrained_file_access")
     sd = settings.get("staticdir")
     if not sd:
-        raise FileAccessDenied("File access disabled.")
+        raise exceptions.FileAccessDenied("File access disabled.")
     sd = os.path.normpath(os.path.abspath(sd))
     s = s[1:]
     s = os.path.expanduser(s)
     s = os.path.normpath(os.path.abspath(os.path.join(sd, s)))
     if not uf and not s.startswith(sd):
-        raise FileAccessDenied("File access outside of configured directory")
+        raise exceptions.FileAccessDenied("File access outside of configured directory")
     if not os.path.isfile(s):
-        raise FileAccessDenied("File not readable")
+        raise exceptions.FileAccessDenied("File not readable")
     return file(s, "rb").read()
-
-
-def parse_response(s):
-    """
-        May raise ParseException
-    """
-    try:
-        s = s.decode("ascii")
-    except UnicodeError:
-        raise ParseException("Spec must be valid ASCII.", 0, 0)
-    try:
-        return Response.expr().parseString(s, parseAll=True)[0]
-    except pp.ParseException, v:
-        raise ParseException(v.msg, v.line, v.col)
-
-
-def parse_requests(s):
-    """
-        May raise ParseException
-    """
-    try:
-        s = s.decode("ascii")
-    except UnicodeError:
-        raise ParseException("Spec must be valid ASCII.", 0, 0)
-    try:
-        return pp.OneOrMore(
-            pp.Or(
-                [
-                    WebsocketFrame.expr(),
-                    Request.expr(),
-                ]
-            )
-        ).parseString(s, parseAll=True)
-    except pp.ParseException, v:
-        raise ParseException(v.msg, v.line, v.col)

libpathod/language/exceptions.py

@@ -0,0 +1,21 @@
+
+class RenderError(Exception):
+    pass
+
+
+class FileAccessDenied(RenderError):
+    pass
+
+
+class ParseException(Exception):
+    def __init__(self, msg, s, col):
+        Exception.__init__(self)
+        self.msg = msg
+        self.s = s
+        self.col = col
+
+    def marked(self):
+        return "%s\n%s"%(self.s, " " * (self.col - 1) + "^")
+
+    def __str__(self):
+        return "%s at char %s"%(self.msg, self.col)

libpathod/language/generators.py

@@ -0,0 +1,99 @@
+import string
+import random
+import mmap
+
+DATATYPES = dict(
+    ascii_letters = string.ascii_letters,
+    ascii_lowercase = string.ascii_lowercase,
+    ascii_uppercase = string.ascii_uppercase,
+    digits = string.digits,
+    hexdigits = string.hexdigits,
+    octdigits = string.octdigits,
+    punctuation = string.punctuation,
+    whitespace = string.whitespace,
+    ascii = string.printable,
+    bytes = "".join(chr(i) for i in range(256))
+)
+
+
+class TransformGenerator:
+    """
+        Perform a byte-by-byte transform another generator - that is, for each
+        input byte, the transformation must produce one output byte.
+
+        gen: A generator to wrap
+        transform: A function (offset, data) -> transformed
+    """
+    def __init__(self, gen, transform):
+        self.gen = gen
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.gen)
+
+    def __getitem__(self, x):
+        d = self.gen.__getitem__(x)
+        return self.transform(x, d)
+
+    def __getslice__(self, a, b):
+        d = self.gen.__getslice__(a, b)
+        return self.transform(a, d)
+
+    def __repr__(self):
+        return "'%s'"%self.gen
+
+
+class LiteralGenerator:
+    def __init__(self, s):
+        self.s = s
+
+    def __len__(self):
+        return len(self.s)
+
+    def __getitem__(self, x):
+        return self.s.__getitem__(x)
+
+    def __getslice__(self, a, b):
+        return self.s.__getslice__(a, b)
+
+    def __repr__(self):
+        return "'%s'"%self.s
+
+
+class RandomGenerator:
+    def __init__(self, dtype, length):
+        self.dtype = dtype
+        self.length = length
+
+    def __len__(self):
+        return self.length
+
+    def __getitem__(self, x):
+        return random.choice(DATATYPES[self.dtype])
+
+    def __getslice__(self, a, b):
+        b = min(b, self.length)
+        chars = DATATYPES[self.dtype]
+        return "".join(random.choice(chars) for x in range(a, b))
+
+    def __repr__(self):
+        return "%s random from %s"%(self.length, self.dtype)
+
+
+class FileGenerator:
+    def __init__(self, path):
+        self.path = path
+        self.fp = file(path, "rb")
+        self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ)
+
+    def __len__(self):
+        return len(self.map)
+
+    def __getitem__(self, x):
+        return self.map.__getitem__(x)
+
+    def __getslice__(self, a, b):
+        return self.map.__getslice__(a, b)
+
+    def __repr__(self):
+        return "<%s"%self.path

libpathod/language/http.py

@@ -0,0 +1,267 @@
+
+import abc
+
+import contrib.pyparsing as pp
+
+import netlib.websockets
+from netlib import http_status
+from . import base, generators, exceptions
+
+
+def get_header(val, headers):
+    """
+        Header keys may be Values, so we have to "generate" them as we try the
+        match.
+    """
+    for h in headers:
+        k = h.key.get_generator({})
+        if len(k) == len(val) and k[:].lower() == val.lower():
+            return h
+    return None
+
+
+class _HTTPMessage(base._Message):
+    version = "HTTP/1.1"
+
+    @abc.abstractmethod
+    def preamble(self, settings): # pragma: no cover
+        pass
+
+    def values(self, settings):
+        vals = self.preamble(settings)
+        vals.append("\r\n")
+        for h in self.headers:
+            vals.extend(h.values(settings))
+        vals.append("\r\n")
+        if self.body:
+            vals.append(self.body.value.get_generator(settings))
+        return vals
+
+
+class Response(_HTTPMessage):
+    comps = (
+        base.Body,
+        base.Header,
+        base.PauseAt,
+        base.DisconnectAt,
+        base.InjectAt,
+        base.ShortcutContentType,
+        base.ShortcutLocation,
+        base.Raw,
+        base.Reason
+    )
+    logattrs = ["code", "reason", "version", "body"]
+
+    @property
+    def ws(self):
+        return self.tok(base.WS)
+
+    @property
+    def code(self):
+        return self.tok(base.Code)
+
+    @property
+    def reason(self):
+        return self.tok(base.Reason)
+
+    def preamble(self, settings):
+        l = [self.version, " "]
+        l.extend(self.code.values(settings))
+        code = int(self.code.code)
+        l.append(" ")
+        if self.reason:
+            l.extend(self.reason.values(settings))
+        else:
+            l.append(
+                generators.LiteralGenerator(
+                    http_status.RESPONSES.get(
+                        code,
+                        "Unknown code"
+                    )
+                )
+            )
+        return l
+
+    def resolve(self, settings, msg=None):
+        tokens = self.tokens[:]
+        if self.ws:
+            if not settings.websocket_key:
+                raise exceptions.RenderError(
+                    "No websocket key - have we seen a client handshake?"
+                )
+            if not self.code:
+                tokens.insert(
+                    1,
+                    base.Code(101)
+                )
+            hdrs = netlib.websockets.server_handshake_headers(
+                settings.websocket_key
+            )
+            for i in hdrs.lst:
+                if not get_header(i[0], self.headers):
+                    tokens.append(
+                        base.Header(
+                            base.ValueLiteral(i[0]),
+                            base.ValueLiteral(i[1]))
+                    )
+        if not self.raw:
+            if not get_header("Content-Length", self.headers):
+                if not self.body:
+                    length = 0
+                else:
+                    length = len(self.body.value.get_generator(settings))
+                tokens.append(
+                    base.Header(
+                        base.ValueLiteral("Content-Length"),
+                        base.ValueLiteral(str(length)),
+                    )
+                )
+        intermediate = self.__class__(tokens)
+        return self.__class__(
+            [i.resolve(settings, intermediate) for i in tokens]
+        )
+
+    @classmethod
+    def expr(klass):
+        parts = [i.expr() for i in klass.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                pp.MatchFirst(
+                    [
+                        base.WS.expr() + pp.Optional(
+                            base.Sep + base.Code.expr()
+                        ),
+                        base.Code.expr(),
+                    ]
+                ),
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(klass)
+        return resp
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+
+class Request(_HTTPMessage):
+    comps = (
+        base.Body,
+        base.Header,
+        base.PauseAt,
+        base.DisconnectAt,
+        base.InjectAt,
+        base.ShortcutContentType,
+        base.ShortcutUserAgent,
+        base.Raw,
+        base.PathodSpec,
+    )
+    logattrs = ["method", "path", "body"]
+
+    @property
+    def ws(self):
+        return self.tok(base.WS)
+
+    @property
+    def method(self):
+        return self.tok(base.Method)
+
+    @property
+    def path(self):
+        return self.tok(base.Path)
+
+    @property
+    def pathodspec(self):
+        return self.tok(base.PathodSpec)
+
+    def preamble(self, settings):
+        v = self.method.values(settings)
+        v.append(" ")
+        v.extend(self.path.values(settings))
+        if self.pathodspec:
+            v.append(self.pathodspec.parsed.spec())
+        v.append(" ")
+        v.append(self.version)
+        return v
+
+    def resolve(self, settings, msg=None):
+        tokens = self.tokens[:]
+        if self.ws:
+            if not self.method:
+                tokens.insert(
+                    1,
+                    base.Method("get")
+                )
+            for i in netlib.websockets.client_handshake_headers().lst:
+                if not get_header(i[0], self.headers):
+                    tokens.append(
+                        base.Header(
+                            base.ValueLiteral(i[0]),
+                            base.ValueLiteral(i[1])
+                        )
+                    )
+        if not self.raw:
+            if not get_header("Content-Length", self.headers):
+                if self.body:
+                    length = len(self.body.value.get_generator(settings))
+                    tokens.append(
+                        base.Header(
+                            base.ValueLiteral("Content-Length"),
+                            base.ValueLiteral(str(length)),
+                        )
+                    )
+            if settings.request_host:
+                if not get_header("Host", self.headers):
+                    tokens.append(
+                        base.Header(
+                            base.ValueLiteral("Host"),
+                            base.ValueLiteral(settings.request_host)
+                        )
+                    )
+        intermediate = self.__class__(tokens)
+        return self.__class__(
+            [i.resolve(settings, intermediate) for i in tokens]
+        )
+
+    @classmethod
+    def expr(klass):
+        parts = [i.expr() for i in klass.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                pp.MatchFirst(
+                    [
+                        base.WS.expr() + pp.Optional(
+                            base.Sep + base.Method.expr()
+                        ),
+                        base.Method.expr(),
+                    ]
+                ),
+                base.Sep,
+                base.Path.expr(),
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(klass)
+        return resp
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+
+class PathodErrorResponse(Response):
+    pass
+
+
+def make_error_response(reason, body=None):
+    tokens = [
+        base.Code("800"),
+        base.Header(
+            base.ValueLiteral("Content-Type"),
+            base.ValueLiteral("text/plain")
+        ),
+        base.Reason(base.ValueLiteral(reason)),
+        base.Body(base.ValueLiteral("pathod error: " + (body or reason))),
+    ]
+    return PathodErrorResponse(tokens)

libpathod/language/websockets.py

@@ -0,0 +1,59 @@
+
+import netlib.websockets
+import contrib.pyparsing as pp
+from . import base, generators
+
+
+class WebsocketFrame(base._Message):
+    comps = (
+        base.Body,
+        base.PauseAt,
+        base.DisconnectAt,
+        base.InjectAt
+    )
+    logattrs = ["body"]
+
+    @classmethod
+    def expr(klass):
+        parts = [i.expr() for i in klass.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                base.WF.expr(),
+                base.Sep,
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(klass)
+        return resp
+
+    def values(self, settings):
+        vals = []
+        if self.body:
+            bodygen = self.body.value.get_generator(settings)
+            length = len(self.body.value.get_generator(settings))
+        else:
+            bodygen = None
+            length = 0
+        frame = netlib.websockets.FrameHeader(
+            mask = True,
+            payload_length = length
+        )
+        vals = [frame.to_bytes()]
+        if self.body:
+            masker = netlib.websockets.Masker(frame.masking_key)
+            vals.append(
+                generators.TransformGenerator(
+                    bodygen,
+                    masker.mask
+                )
+            )
+        return vals
+
+    def resolve(self, settings, msg=None):
+        return self.__class__(
+            [i.resolve(settings, msg) for i in self.tokens]
+        )
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])

libpathod/language/writer.py

@@ -0,0 +1,61 @@
+import time
+import netlib.tcp
+
+BLOCKSIZE = 1024
+
+
+def send_chunk(fp, val, blocksize, start, end):
+    """
+        (start, end): Inclusive lower bound, exclusive upper bound.
+    """
+    for i in range(start, end, blocksize):
+        fp.write(
+            val[i:min(i + blocksize, end)]
+        )
+    return end - start
+
+
+def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE):
+    """
+        vals: A list of values, which may be strings or Value objects.
+
+        actions: A list of (offset, action, arg) tuples. Action may be "pause"
+        or "disconnect".
+
+        Both vals and actions are in reverse order, with the first items last.
+
+        Return True if connection should disconnect.
+    """
+    sofar = 0
+    try:
+        while vals:
+            v = vals.pop()
+            offset = 0
+            while actions and actions[-1][0] < (sofar + len(v)):
+                a = actions.pop()
+                offset += send_chunk(
+                    fp,
+                    v,
+                    blocksize,
+                    offset,
+                    a[0] - sofar - offset
+                )
+                if a[1] == "pause":
+                    time.sleep(a[2])
+                elif a[1] == "disconnect":
+                    return True
+                elif a[1] == "inject":
+                    send_chunk(fp, a[2], blocksize, 0, len(a[2]))
+            send_chunk(fp, v, blocksize, offset, len(v))
+            sofar += len(v)
+        # Remainders
+        while actions:
+            a = actions.pop()
+            if a[1] == "pause":
+                time.sleep(a[2])
+            elif a[1] == "disconnect":
+                return True
+            elif a[1] == "inject":
+                send_chunk(fp, a[2], blocksize, 0, len(a[2]))
+    except netlib.tcp.NetLibDisconnect: # pragma: no cover
+        return True

libpathod/pathoc.py

@@ -12,7 +12,8 @@ import OpenSSL.crypto
 from netlib import tcp, http, certutils, websockets
 import netlib.utils
 
-import language
+import language.http
+import language.websockets
 import utils
 
 
@@ -346,7 +347,7 @@ class Pathoc(tcp.TCPClient):
         """
             Performs a single request.
 
-            r: A language.Request object, or a string representing one request.
+            r: A language.http.Request object, or a string representing one request.
 
             Returns Response if we have a non-ignored response.
 
@@ -385,7 +386,7 @@ class Pathoc(tcp.TCPClient):
         """
             Performs a single request.
 
-            r: A language.Request object, or a string representing one request.
+            r: A language.http.Request object, or a string representing one request.
 
             Returns Response if we have a non-ignored response.
 
@@ -393,12 +394,12 @@ class Pathoc(tcp.TCPClient):
         """
         if isinstance(r, basestring):
             r = language.parse_requests(r)[0]
-        if isinstance(r, language.Request):
+        if isinstance(r, language.http.Request):
             if r.ws:
                 return self.websocket_start(r, self.websocket_get_frame)
             else:
                 return self.http(r)
-        elif isinstance(r, language.WebsocketFrame):
+        elif isinstance(r, language.websockets.WebsocketFrame):
             self.websocket_send_frame(r)
 
 

libpathod/pathod.py

@@ -8,6 +8,7 @@ from netlib import tcp, http, wsgi, certutils, websockets
 import netlib.utils
 
 from . import version, app, language, utils
+import language.http
 
 
 DEFAULT_CERT_DOMAIN = "pathod.net"
@@ -75,7 +76,7 @@ class PathodHandler(tcp.BaseHandler):
             crafted, self.settings
         )
         if error:
-            err = language.make_error_response(error)
+            err = language.http.make_error_response(error)
             language.serve(err, self.wfile, self.settings)
             log = dict(
                 type="error",
@@ -83,7 +84,7 @@ class PathodHandler(tcp.BaseHandler):
             )
             return False, log
 
-        if self.server.explain and not isinstance(crafted, language.PathodErrorResponse):
+        if self.server.explain and not isinstance(crafted, language.http.PathodErrorResponse):
             crafted = crafted.freeze(self.settings)
             self.info(">> Spec: %s" % crafted.spec())
         response_log = language.serve(
@@ -212,7 +213,7 @@ class PathodHandler(tcp.BaseHandler):
                 crafted = language.parse_response(spec)
             except language.ParseException, v:
                 self.info("Parse error: %s" % v.msg)
-                crafted = language.make_error_response(
+                crafted = language.http.make_error_response(
                     "Parse Error",
                     "Error parsing response spec: %s\n" % v.msg + v.marked()
                 )
@@ -220,7 +221,7 @@ class PathodHandler(tcp.BaseHandler):
             self.addlog(retlog)
             return again
         elif self.server.noweb:
-            crafted = language.make_error_response("Access Denied")
+            crafted = language.http.make_error_response("Access Denied")
             language.serve(crafted, self.wfile, self.settings)
             self.addlog(dict(
                 type="error",
@@ -364,7 +365,8 @@ class Pathod(tcp.TCPServer):
             return "File access denied.", None
         if self.sizelimit and l > self.sizelimit:
             return "Response too large.", None
-        if self.nohang and any([isinstance(i, language.PauseAt) for i in req.actions]):
+        pauses = [isinstance(i, language.base.PauseAt) for i in req.actions]
+        if self.nohang and any(pauses):
             return "Pauses have been disabled.", None
         return None, req
 

libpathod/utils.py

@@ -44,17 +44,6 @@ def parse_size(s):
     raise ValueError("Invalid size specification.")
 
 
-def get_header(val, headers):
-    """
-        Header keys may be Values, so we have to "generate" them as we try the match.
-    """
-    for h in headers:
-        k = h.key.get_generator({})
-        if len(k) == len(val) and k[:].lower() == val.lower():
-            return h
-    return None
-
-
 def parse_anchor_spec(s):
     """
         Return a tuple, or None on error.

test/test_language.py

@@ -1,63 +1,60 @@
 import os
 import cStringIO
-from libpathod import language, utils
+from libpathod import language
+from libpathod.language import generators, base, http, websockets, writer, exceptions
 import tutils
 
 language.TESTING = True
 
 
-def test_quote():
-    assert language.quote("'\\\\'")
-
-
 def parse_request(s):
     return language.parse_requests(s)[0]
 
 
 class TestWS:
     def test_expr(self):
-        v = language.WS("foo")
+        v = base.WS("foo")
         assert v.expr()
         assert v.values(language.Settings())
 
 
 class TestValueNakedLiteral:
     def test_expr(self):
-        v = language.ValueNakedLiteral("foo")
+        v = base.ValueNakedLiteral("foo")
         assert v.expr()
 
     def test_spec(self):
-        v = language.ValueNakedLiteral("foo")
+        v = base.ValueNakedLiteral("foo")
         assert v.spec() == repr(v) == "foo"
 
-        v = language.ValueNakedLiteral("f\x00oo")
+        v = base.ValueNakedLiteral("f\x00oo")
         assert v.spec() == repr(v) == r"f\x00oo"
 
 
 class TestValueLiteral:
     def test_espr(self):
-        v = language.ValueLiteral("foo")
+        v = base.ValueLiteral("foo")
         assert v.expr()
         assert v.val == "foo"
 
-        v = language.ValueLiteral("foo\n")
+        v = base.ValueLiteral("foo\n")
         assert v.expr()
         assert v.val == "foo\n"
         assert repr(v)
 
     def test_spec(self):
-        v = language.ValueLiteral("foo")
+        v = base.ValueLiteral("foo")
         assert v.spec() == r"'foo'"
 
-        v = language.ValueLiteral("f\x00oo")
+        v = base.ValueLiteral("f\x00oo")
         assert v.spec() == repr(v) == r"'f\x00oo'"
 
-        v = language.ValueLiteral("\"")
+        v = base.ValueLiteral("\"")
         assert v.spec() == repr(v) == '\'"\''
 
     def roundtrip(self, spec):
-        e = language.ValueLiteral.expr()
-        v = language.ValueLiteral(spec)
+        e = base.ValueLiteral.expr()
+        v = base.ValueLiteral(spec)
         v2 = e.parseString(v.spec())
         assert v.val == v2[0].val
         assert v.spec() == v2[0].spec()
@@ -73,56 +70,56 @@ class TestValueLiteral:
 
 class TestValueGenerate:
     def test_basic(self):
-        v = language.Value.parseString("@10b")[0]
+        v = base.Value.parseString("@10b")[0]
         assert v.usize == 10
         assert v.unit == "b"
         assert v.bytes() == 10
-        v = language.Value.parseString("@10")[0]
+        v = base.Value.parseString("@10")[0]
         assert v.unit == "b"
-        v = language.Value.parseString("@10k")[0]
+        v = base.Value.parseString("@10k")[0]
         assert v.bytes() == 10240
-        v = language.Value.parseString("@10g")[0]
+        v = base.Value.parseString("@10g")[0]
         assert v.bytes() == 1024**3 * 10
 
-        v = language.Value.parseString("@10g,digits")[0]
+        v = base.Value.parseString("@10g,digits")[0]
         assert v.datatype == "digits"
         g = v.get_generator({})
         assert g[:100]
 
-        v = language.Value.parseString("@10,digits")[0]
+        v = base.Value.parseString("@10,digits")[0]
         assert v.unit == "b"
         assert v.datatype == "digits"
 
     def test_spec(self):
-        v = language.ValueGenerate(1, "b", "bytes")
+        v = base.ValueGenerate(1, "b", "bytes")
         assert v.spec() == repr(v) == "@1"
 
-        v = language.ValueGenerate(1, "k", "bytes")
+        v = base.ValueGenerate(1, "k", "bytes")
         assert v.spec() == repr(v) == "@1k"
 
-        v = language.ValueGenerate(1, "k", "ascii")
+        v = base.ValueGenerate(1, "k", "ascii")
         assert v.spec() == repr(v) == "@1k,ascii"
 
-        v = language.ValueGenerate(1, "b", "ascii")
+        v = base.ValueGenerate(1, "b", "ascii")
         assert v.spec() == repr(v) == "@1,ascii"
 
     def test_freeze(self):
-        v = language.ValueGenerate(100, "b", "ascii")
+        v = base.ValueGenerate(100, "b", "ascii")
         f = v.freeze(language.Settings())
         assert len(f.val) == 100
 
 
 class TestValueFile:
     def test_file_value(self):
-        v = language.Value.parseString("<'one two'")[0]
+        v = base.Value.parseString("<'one two'")[0]
         assert str(v)
         assert v.path == "one two"
 
-        v = language.Value.parseString("<path")[0]
+        v = base.Value.parseString("<path")[0]
         assert v.path == "path"
 
     def test_access_control(self):
-        v = language.Value.parseString("<path")[0]
+        v = base.Value.parseString("<path")[0]
         with tutils.tmpdir() as t:
             p = os.path.join(t, "path")
             with open(p, "wb") as f:
@@ -130,9 +127,9 @@ class TestValueFile:
 
             assert v.get_generator(language.Settings(staticdir=t))
 
-            v = language.Value.parseString("<path2")[0]
+            v = base.Value.parseString("<path2")[0]
             tutils.raises(
-                language.FileAccessDenied,
+                exceptions.FileAccessDenied,
                 v.get_generator,
                 language.Settings(staticdir=t)
             )
@@ -142,7 +139,7 @@ class TestValueFile:
                 language.Settings()
             )
 
-            v = language.Value.parseString("</outside")[0]
+            v = base.Value.parseString("</outside")[0]
             tutils.raises(
                 "outside",
                 v.get_generator,
@@ -150,24 +147,24 @@ class TestValueFile:
             )
 
     def test_spec(self):
-        v = language.Value.parseString("<'one two'")[0]
-        v2 = language.Value.parseString(v.spec())[0]
+        v = base.Value.parseString("<'one two'")[0]
+        v2 = base.Value.parseString(v.spec())[0]
         assert v2.path == "one two"
 
     def test_freeze(self):
-        v = language.Value.parseString("<'one two'")[0]
+        v = base.Value.parseString("<'one two'")[0]
         v2 = v.freeze({})
         assert v2.path == v.path
 
 
 class TestMisc:
     def test_generators(self):
-        v = language.Value.parseString("'val'")[0]
+        v = base.Value.parseString("'val'")[0]
         g = v.get_generator({})
         assert g[:] == "val"
 
     def test_randomgenerator(self):
-        g = language.RandomGenerator("bytes", 100)
+        g = generators.RandomGenerator("bytes", 100)
         assert repr(g)
         assert len(g[:10]) == 10
         assert len(g[1:10]) == 9
@@ -176,7 +173,7 @@ class TestMisc:
         assert g[0]
 
     def test_literalgenerator(self):
-        g = language.LiteralGenerator("one")
+        g = generators.LiteralGenerator("one")
         assert repr(g)
         assert g[:] == "one"
         assert g[1] == "n"
@@ -187,7 +184,7 @@ class TestMisc:
             f = open(path, "wb")
             f.write("x"*10000)
             f.close()
-            g = language.FileGenerator(path)
+            g = generators.FileGenerator(path)
             assert len(g) == 10000
             assert g[0] == "x"
             assert g[-1] == "x"
@@ -196,15 +193,15 @@ class TestMisc:
             del g  # remove all references to FileGenerator instance to close the file handle.
 
     def test_value(self):
-        assert language.Value.parseString("'val'")[0].val == "val"
-        assert language.Value.parseString('"val"')[0].val == "val"
-        assert language.Value.parseString('"\'val\'"')[0].val == "'val'"
+        assert base.Value.parseString("'val'")[0].val == "val"
+        assert base.Value.parseString('"val"')[0].val == "val"
+        assert base.Value.parseString('"\'val\'"')[0].val == "'val'"
 
     def test_path(self):
-        e = language.Path.expr()
+        e = base.Path.expr()
         assert e.parseString('"/foo"')[0].value.val == "/foo"
 
-        v = language.Path("/foo")
+        v = base.Path("/foo")
         assert v.value.val == "/foo"
 
         v = e.parseString("@100")[0]
@@ -217,7 +214,7 @@ class TestMisc:
         assert s == v.expr().parseString(s)[0].spec()
 
     def test_method(self):
-        e = language.Method.expr()
+        e = base.Method.expr()
         assert e.parseString("get")[0].value.val == "GET"
         assert e.parseString("'foo'")[0].value.val == "foo"
         assert e.parseString("'get'")[0].value.val == "get"
@@ -237,13 +234,13 @@ class TestMisc:
         assert v2.value.val == v3.value.val
 
     def test_raw(self):
-        e = language.Raw.expr().parseString("r")[0]
+        e = base.Raw.expr().parseString("r")[0]
         assert e
         assert e.spec() == "r"
         assert e.freeze({}).spec() == "r"
 
     def test_body(self):
-        e = language.Body.expr()
+        e = base.Body.expr()
         v = e.parseString("b'foo'")[0]
         assert v.value.val == "foo"
 
@@ -261,7 +258,7 @@ class TestMisc:
         assert s == e.parseString(s)[0].spec()
 
     def test_pathodspec(self):
-        e = language.PathodSpec.expr()
+        e = base.PathodSpec.expr()
         v = e.parseString("s'200'")[0]
         assert v.value.val == "200"
         tutils.raises(
@@ -276,8 +273,8 @@ class TestMisc:
         assert "@1" not in f.spec()
 
     def test_pathodspec_freeze(self):
-        e = language.PathodSpec(
-            language.ValueLiteral(
+        e = base.PathodSpec(
+            base.ValueLiteral(
                 "200:b'foo':i10,'\\''".encode(
                     "string_escape"
                 )
@@ -287,7 +284,7 @@ class TestMisc:
         assert e.values({})
 
     def test_code(self):
-        e = language.Code.expr()
+        e = base.Code.expr()
         v = e.parseString("200")[0]
         assert v.string() == "200"
         assert v.spec() == "200"
@@ -295,7 +292,7 @@ class TestMisc:
         assert v.freeze({}).code == v.code
 
     def test_reason(self):
-        e = language.Reason.expr()
+        e = base.Reason.expr()
         v = e.parseString("m'msg'")[0]
         assert v.value.val == "msg"
 
@@ -309,13 +306,13 @@ class TestMisc:
 
     def test_internal_response(self):
         d = cStringIO.StringIO()
-        s = language.make_error_response("foo")
+        s = http.make_error_response("foo")
         language.serve(s, d, {})
 
 
 class TestHeaders:
     def test_header(self):
-        e = language.Header.expr()
+        e = base.Header.expr()
         v = e.parseString("h'foo'='bar'")[0]
         assert v.key.val == "foo"
         assert v.value.val == "bar"
@@ -328,7 +325,7 @@ class TestHeaders:
         assert s == e.parseString(s)[0].spec()
 
     def test_header_freeze(self):
-        e = language.Header.expr()
+        e = base.Header.expr()
         v = e.parseString("h@10=@10'")[0]
         v2 = v.freeze({})
         v3 = v2.freeze({})
@@ -336,7 +333,7 @@ class TestHeaders:
         assert v2.value.val == v3.value.val
 
     def test_ctype_shortcut(self):
-        e = language.ShortcutContentType.expr()
+        e = base.ShortcutContentType.expr()
         v = e.parseString("c'foo'")[0]
         assert v.key.val == "Content-Type"
         assert v.value.val == "foo"
@@ -344,14 +341,14 @@ class TestHeaders:
         s = v.spec()
         assert s == e.parseString(s)[0].spec()
 
-        e = language.ShortcutContentType.expr()
+        e = base.ShortcutContentType.expr()
         v = e.parseString("c@100")[0]
         v2 = v.freeze({})
         v3 = v2.freeze({})
         assert v2.value.val == v3.value.val
 
     def test_location_shortcut(self):
-        e = language.ShortcutLocation.expr()
+        e = base.ShortcutLocation.expr()
         v = e.parseString("l'foo'")[0]
         assert v.key.val == "Location"
         assert v.value.val == "foo"
@@ -359,7 +356,7 @@ class TestHeaders:
         s = v.spec()
         assert s == e.parseString(s)[0].spec()
 
-        e = language.ShortcutLocation.expr()
+        e = base.ShortcutLocation.expr()
         v = e.parseString("l@100")[0]
         v2 = v.freeze({})
         v3 = v2.freeze({})
@@ -375,7 +372,7 @@ class TestHeaders:
 
 class TestShortcutUserAgent:
     def test_location_shortcut(self):
-        e = language.ShortcutUserAgent.expr()
+        e = base.ShortcutUserAgent.expr()
         v = e.parseString("ua")[0]
         assert "Android" in str(v.value)
         assert v.spec() == "ua"
@@ -394,9 +391,9 @@ class TestShortcutUserAgent:
 
 class Test_Action:
     def test_cmp(self):
-        a = language.DisconnectAt(0)
-        b = language.DisconnectAt(1)
-        c = language.DisconnectAt(0)
+        a = base.DisconnectAt(0)
+        b = base.DisconnectAt(1)
+        c = base.DisconnectAt(0)
         assert a < b
         assert a == c
         l = [b, a]
@@ -405,16 +402,16 @@ class Test_Action:
 
     def test_resolve(self):
         r = parse_request('GET:"/foo"')
-        e = language.DisconnectAt("r")
+        e = base.DisconnectAt("r")
         ret = e.resolve({}, r)
         assert isinstance(ret.offset, int)
 
     def test_repr(self):
-        e = language.DisconnectAt("r")
+        e = base.DisconnectAt("r")
         assert repr(e)
 
     def test_freeze(self):
-        l = language.DisconnectAt(5)
+        l = base.DisconnectAt(5)
         assert l.freeze({}).spec() == l.spec()
 
 
@@ -426,21 +423,21 @@ class TestDisconnects:
         assert a.spec() == "dr"
 
     def test_at(self):
-        e = language.DisconnectAt.expr()
+        e = base.DisconnectAt.expr()
         v = e.parseString("d0")[0]
-        assert isinstance(v, language.DisconnectAt)
+        assert isinstance(v, base.DisconnectAt)
         assert v.offset == 0
 
         v = e.parseString("d100")[0]
         assert v.offset == 100
 
-        e = language.DisconnectAt.expr()
+        e = base.DisconnectAt.expr()
         v = e.parseString("dr")[0]
         assert v.offset == "r"
 
     def test_spec(self):
-        assert language.DisconnectAt("r").spec() == "dr"
-        assert language.DisconnectAt(10).spec() == "d10"
+        assert base.DisconnectAt("r").spec() == "dr"
+        assert base.DisconnectAt(10).spec() == "d10"
 
 
 class TestInject:
@@ -454,11 +451,11 @@ class TestInject:
         assert a.offset == "a"
 
     def test_at(self):
-        e = language.InjectAt.expr()
+        e = base.InjectAt.expr()
         v = e.parseString("i0,'foo'")[0]
         assert v.value.val == "foo"
         assert v.offset == 0
-        assert isinstance(v, language.InjectAt)
+        assert isinstance(v, base.InjectAt)
 
         v = e.parseString("ir,'foo'")[0]
         assert v.offset == "r"
@@ -469,12 +466,12 @@ class TestInject:
         assert language.serve(r, s, {})
 
     def test_spec(self):
-        e = language.InjectAt.expr()
+        e = base.InjectAt.expr()
         v = e.parseString("i0,'foo'")[0]
         assert v.spec() == 'i0,"foo"'
 
     def test_spec(self):
-        e = language.InjectAt.expr()
+        e = base.InjectAt.expr()
         v = e.parseString("i0,@100")[0]
         v2 = v.freeze({})
         v3 = v2.freeze({})
@@ -483,7 +480,7 @@ class TestInject:
 
 class TestPauses:
     def test_parse_response(self):
-        e = language.PauseAt.expr()
+        e = base.PauseAt.expr()
         v = e.parseString("p10,10")[0]
         assert v.seconds == 10
         assert v.offset == 10
@@ -502,12 +499,12 @@ class TestPauses:
         assert r.actions[0].spec() == "p10,10"
 
     def test_spec(self):
-        assert language.PauseAt("r", 5).spec() == "pr,5"
-        assert language.PauseAt(0, 5).spec() == "p0,5"
-        assert language.PauseAt(0, "f").spec() == "p0,f"
+        assert base.PauseAt("r", 5).spec() == "pr,5"
+        assert base.PauseAt(0, 5).spec() == "p0,5"
+        assert base.PauseAt(0, "f").spec() == "p0,f"
 
     def test_freeze(self):
-        l = language.PauseAt("r", 5)
+        l = base.PauseAt("r", 5)
         assert l.freeze({}).spec() == l.spec()
 
 
@@ -567,7 +564,7 @@ class TestRequest:
         r = language.parse_requests(l)
         assert len(r) == 1
         assert len(r[0].tokens) == 3
-        assert isinstance(r[0].tokens[2], language.PathodSpec)
+        assert isinstance(r[0].tokens[2], base.PathodSpec)
         assert r[0].values({})
 
     def test_render(self):
@@ -625,21 +622,21 @@ class TestRequest:
         r = parse_request('ws:/path/')
         res = r.resolve(language.Settings())
         assert res.method.string().lower() == "get"
-        assert res.tok(language.Path).value.val == "/path/"
-        assert res.tok(language.Method).value.val.lower() == "get"
-        assert utils.get_header("Upgrade", res.headers).value.val == "websocket"
+        assert res.tok(base.Path).value.val == "/path/"
+        assert res.tok(base.Method).value.val.lower() == "get"
+        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
 
         r = parse_request('ws:put:/path/')
         res = r.resolve(language.Settings())
         assert r.method.string().lower() == "put"
-        assert res.tok(language.Path).value.val == "/path/"
-        assert res.tok(language.Method).value.val.lower() == "put"
-        assert utils.get_header("Upgrade", res.headers).value.val == "websocket"
+        assert res.tok(base.Path).value.val == "/path/"
+        assert res.tok(base.Method).value.val.lower() == "put"
+        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
 
 
 class TestWebsocketFrame:
     def test_spec(self):
-        e = language.WebsocketFrame.expr()
+        e = websockets.WebsocketFrame.expr()
         wf = e.parseString("wf:b'foo'")
         assert wf
 
@@ -656,45 +653,45 @@ class TestWriteValues:
         v = "foobarfoobar"
         for bs in range(1, len(v) + 2):
             s = cStringIO.StringIO()
-            language.send_chunk(s, v, bs, 0, len(v))
+            writer.send_chunk(s, v, bs, 0, len(v))
             assert s.getvalue() == v
             for start in range(len(v)):
                 for end in range(len(v)):
                     s = cStringIO.StringIO()
-                    language.send_chunk(s, v, bs, start, end)
+                    writer.send_chunk(s, v, bs, start, end)
                     assert s.getvalue() == v[start:end]
 
     def test_write_values_inject(self):
         tst = "foo"
 
         s = cStringIO.StringIO()
-        language.write_values(s, [tst], [(0, "inject", "aaa")], blocksize=5)
+        writer.write_values(s, [tst], [(0, "inject", "aaa")], blocksize=5)
         assert s.getvalue() == "aaafoo"
 
         s = cStringIO.StringIO()
-        language.write_values(s, [tst], [(1, "inject", "aaa")], blocksize=5)
+        writer.write_values(s, [tst], [(1, "inject", "aaa")], blocksize=5)
         assert s.getvalue() == "faaaoo"
 
         s = cStringIO.StringIO()
-        language.write_values(s, [tst], [(1, "inject", "aaa")], blocksize=5)
+        writer.write_values(s, [tst], [(1, "inject", "aaa")], blocksize=5)
         assert s.getvalue() == "faaaoo"
 
     def test_write_values_disconnects(self):
         s = cStringIO.StringIO()
         tst = "foo" * 100
-        language.write_values(s, [tst], [(0, "disconnect")], blocksize=5)
+        writer.write_values(s, [tst], [(0, "disconnect")], blocksize=5)
         assert not s.getvalue()
 
     def test_write_values(self):
         tst = "foobarvoing"
         s = cStringIO.StringIO()
-        language.write_values(s, [tst], [])
+        writer.write_values(s, [tst], [])
         assert s.getvalue() == tst
 
         for bs in range(1, len(tst) + 2):
             for off in range(len(tst)):
                 s = cStringIO.StringIO()
-                language.write_values(
+                writer.write_values(
                     s, [tst], [(off, "disconnect")], blocksize=bs
                 )
                 assert s.getvalue() == tst[:off]
@@ -703,20 +700,20 @@ class TestWriteValues:
         tst = "".join(str(i) for i in range(10))
         for i in range(2, 10):
             s = cStringIO.StringIO()
-            language.write_values(
+            writer.write_values(
                 s, [tst], [(2, "pause", 0), (1, "pause", 0)], blocksize=i
             )
             assert s.getvalue() == tst
 
         for i in range(2, 10):
             s = cStringIO.StringIO()
-            language.write_values(s, [tst], [(1, "pause", 0)], blocksize=i)
+            writer.write_values(s, [tst], [(1, "pause", 0)], blocksize=i)
             assert s.getvalue() == tst
 
         tst = ["".join(str(i) for i in range(10))] * 5
         for i in range(2, 10):
             s = cStringIO.StringIO()
-            language.write_values(s, tst[:], [(1, "pause", 0)], blocksize=i)
+            writer.write_values(s, tst[:], [(1, "pause", 0)], blocksize=i)
             assert s.getvalue() == "".join(tst)
 
     def test_write_values_after(self):
@@ -816,7 +813,7 @@ class TestResponse:
 
     def test_parse_header(self):
         r = language.parse_response('400:h"foo"="bar"')
-        assert utils.get_header("foo", r.headers)
+        assert http.get_header("foo", r.headers)
 
     def test_parse_pause_before(self):
         r = language.parse_response("400:p0,10")
@@ -854,28 +851,28 @@ class TestResponse:
 
 
 def test_read_file():
-    tutils.raises(language.FileAccessDenied, language.read_file, {}, "=/foo")
+    tutils.raises(exceptions.FileAccessDenied, base.read_file, {}, "=/foo")
     p = tutils.test_data.path("data")
     d = dict(staticdir=p)
-    assert language.read_file(d, "+./file").strip() == "testfile"
-    assert language.read_file(d, "+file").strip() == "testfile"
+    assert base.read_file(d, "+./file").strip() == "testfile"
+    assert base.read_file(d, "+file").strip() == "testfile"
     tutils.raises(
-        language.FileAccessDenied,
-        language.read_file,
+        exceptions.FileAccessDenied,
+        base.read_file,
         d,
         "+./nonexistent"
     )
     tutils.raises(
-        language.FileAccessDenied,
-        language.read_file,
+        exceptions.FileAccessDenied,
+        base.read_file,
         d,
         "+/nonexistent"
     )
     tutils.raises(
-        language.FileAccessDenied,
-        language.read_file,
+        exceptions.FileAccessDenied,
+        base.read_file,
         d,
         "+../test_language.py"
     )
     d["unconstrained_file_access"] = True
-    assert language.read_file(d, "+../test_language.py")
+    assert base.read_file(d, "+../test_language.py")