diff --git a/libpathod/language.py b/libpathod/language.py index 002c8205c..2969055be 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -1,4 +1,10 @@ -import operator, string, random, mmap, os, time, copy +import operator +import string +import random +import mmap +import os +import time +import copy import abc from email.utils import formatdate import contrib.pyparsing as pp @@ -9,7 +15,9 @@ import utils BLOCKSIZE = 1024 TRUNCATE = 1024 -class FileAccessDenied(Exception): pass + +class FileAccessDenied(Exception): + pass class ParseException(Exception): @@ -20,7 +28,7 @@ class ParseException(Exception): self.col = col def marked(self): - return "%s\n%s"%(self.s, " "*(self.col-1) + "^") + return "%s\n%s"%(self.s, " "*(self.col - 1) + "^") def __str__(self): return "%s at char %s"%(self.msg, self.col) @@ -40,7 +48,9 @@ def send_chunk(fp, val, blocksize, start, end): def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE): """ vals: A list of values, which may be strings or Value objects. - actions: A list of (offset, action, arg) tuples. Action may be "pause" or "disconnect". + + actions: A list of (offset, action, arg) tuples. Action may be "pause" + or "disconnect". Both vals and actions are in reverse order, with the first items last. @@ -53,7 +63,13 @@ def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE): offset = 0 while actions and actions[-1][0] < (sofar + len(v)): a = actions.pop() - offset += send_chunk(fp, v, blocksize, offset, a[0]-sofar-offset) + offset += send_chunk( + fp, + v, + blocksize, + offset, + a[0]-sofar-offset + ) if a[1] == "pause": time.sleep(a[2]) elif a[1] == "disconnect": @@ -128,8 +144,18 @@ v_integer = pp.Regex(r"\d+")\ v_literal = pp.MatchFirst( [ - pp.QuotedString("\"", escChar="\\", unquoteResults=True, multiline=True), - pp.QuotedString("'", escChar="\\", unquoteResults=True, multiline=True), + pp.QuotedString( + "\"", + escChar="\\", + unquoteResults=True, + multiline=True + ), + pp.QuotedString( + "'", + escChar="\\", + unquoteResults=True, + multiline=True + ), ] ) @@ -202,6 +228,7 @@ class _Token(object): A specification token. Tokens are immutable. """ __metaclass__ = abc.ABCMeta + @abc.abstractmethod def expr(klass): # pragma: no cover """ @@ -278,7 +305,10 @@ class ValueGenerate(_Token): def expr(klass): e = pp.Literal("@").suppress() + v_integer - u = reduce(operator.or_, [pp.Literal(i) for i in utils.SIZE_UNITS.keys()]) + u = reduce( + operator.or_, + [pp.Literal(i) for i in utils.SIZE_UNITS.keys()] + ) e = e + pp.Optional(u, default=None) s = pp.Literal(",").suppress() @@ -318,7 +348,9 @@ class ValueFile(_Token): s = os.path.expanduser(self.path) s = os.path.normpath(os.path.abspath(os.path.join(sd, s))) if not uf and not s.startswith(sd): - raise FileAccessDenied("File access outside of configured directory") + raise FileAccessDenied( + "File access outside of configured directory" + ) if not os.path.isfile(s): raise FileAccessDenied("File not readable") return FileGenerator(s) @@ -347,12 +379,12 @@ NakedValue = pp.MatchFirst( Offset = pp.MatchFirst( - [ - v_integer, - pp.Literal("r"), - pp.Literal("a") - ] - ) + [ + v_integer, + pp.Literal("r"), + pp.Literal("a") + ] +) class Raw(_Token): @@ -392,11 +424,11 @@ class _Header(_Component): def values(self, settings): return [ - self.key.get_generator(settings), - ": ", - self.value.get_generator(settings), - "\r\n", - ] + self.key.get_generator(settings), + ": ", + self.value.get_generator(settings), + "\r\n", + ] class Header(_Header): @@ -459,7 +491,10 @@ class ShortcutUserAgent(_Header): @classmethod def expr(klass): e = pp.Literal("u").suppress() - u = reduce(operator.or_, [pp.Literal(i[1]) for i in http_uastrings.UASTRINGS]) + u = reduce( + operator.or_, + [pp.Literal(i[1]) for i in http_uastrings.UASTRINGS] + ) e += u | Value return e.setParseAction(lambda x: klass(*x)) @@ -470,7 +505,6 @@ class ShortcutUserAgent(_Header): return ShortcutUserAgent(self.value.freeze(settings)) - class Body(_Component): def __init__(self, value): self.value = value @@ -483,8 +517,8 @@ class Body(_Component): def values(self, settings): return [ - self.value.get_generator(settings), - ] + self.value.get_generator(settings), + ] def spec(self): return "b%s"%(self.value.spec()) @@ -506,8 +540,8 @@ class Path(_Component): def values(self, settings): return [ - self.value.get_generator(settings), - ] + self.value.get_generator(settings), + ] def spec(self): return "%s"%(self.value.spec()) @@ -527,6 +561,7 @@ class Method(_Component): "trace", "connect", ] + def __init__(self, value): # If it's a string, we were passed one of the methods, so we upper-case # it to be canonical. The user can specify a different case by using a @@ -645,11 +680,11 @@ class PauseAt(_Action): e += Offset e += pp.Literal(",").suppress() e += pp.MatchFirst( - [ - v_integer, - pp.Literal("f") - ] - ) + [ + v_integer, + pp.Literal("f") + ] + ) return e.setParseAction(lambda x: klass(*x)) def spec(self): @@ -700,10 +735,10 @@ class InjectAt(_Action): def intermediate(self, settings): return ( - self.offset, - "inject", - self.value.get_generator(settings) - ) + self.offset, + "inject", + self.value.get_generator(settings) + ) def freeze(self, settings): return InjectAt(self.offset, self.value.freeze(settings)) @@ -712,6 +747,7 @@ class InjectAt(_Action): class _Message(object): __metaclass__ = abc.ABCMeta version = "HTTP/1.1" + def __init__(self, tokens): self.tokens = tokens @@ -741,7 +777,8 @@ class _Message(object): def length(self, settings): """ - Calculate the length of the base message without any applied actions. + Calculate the length of the base message without any applied + actions. """ return sum(len(x) for x in self.values(settings)) @@ -754,7 +791,8 @@ class _Message(object): def maximum_length(self, settings): """ - Calculate the maximum length of the base message with all applied actions. + Calculate the maximum length of the base message with all applied + actions. """ l = self.length(settings) for i in self.actions: @@ -786,7 +824,13 @@ class _Message(object): tokens.append( Header( ValueLiteral("Date"), - ValueLiteral(formatdate(timeval=None, localtime=False, usegmt=True)) + ValueLiteral( + formatdate( + timeval=None, + localtime=False, + usegmt=True + ) + ) ) ) intermediate = self.__class__(tokens) @@ -807,7 +851,8 @@ class _Message(object): ret = {} for i in self.logattrs: v = getattr(self, i) - # Careful not to log any VALUE specs without sanitizing them first. We truncate at 1k. + # Careful not to log any VALUE specs without sanitizing them first. + # We truncate at 1k. if hasattr(v, "values"): v = [x[:TRUNCATE] for x in v.values(settings)] v = "".join(v).encode("string_escape") @@ -838,6 +883,7 @@ class _Message(object): Sep = pp.Optional(pp.Literal(":")).suppress() + class Response(_Message): comps = ( Body, @@ -851,6 +897,7 @@ class Response(_Message): Reason ) logattrs = ["code", "reason", "version", "body"] + @property def code(self): return self._get_token(Code) @@ -866,7 +913,14 @@ class Response(_Message): if self.reason: l.extend(self.reason.values(settings)) else: - l.append(LiteralGenerator(http_status.RESPONSES.get(int(self.code.code), "Unknown code"))) + l.append( + LiteralGenerator( + http_status.RESPONSES.get( + int(self.code.code), + "Unknown code" + ) + ) + ) return l @classmethod @@ -897,6 +951,7 @@ class Request(_Message): Raw ) logattrs = ["method", "path", "body"] + @property def method(self): return self._get_token(Method) @@ -944,7 +999,7 @@ def make_error_response(reason, body=None): ] return PathodErrorResponse(tokens) -FILESTART = "+" + def read_file(settings, s): uf = settings.get("unconstrained_file_access") sd = settings.get("staticdir") diff --git a/test/test_language.py b/test/test_language.py index 73b4583f1..18c68caa4 100644 --- a/test/test_language.py +++ b/test/test_language.py @@ -1,4 +1,5 @@ -import os, cStringIO +import os +import cStringIO from libpathod import language, utils import tutils @@ -475,7 +476,6 @@ class TestRequest: assert r.path.string() == "/foo" assert r.actions - l = """ GET @@ -611,6 +611,11 @@ class TestResponse: r = language.parse_response("400:m'msg'") assert language.serve(r, s, {}) + r = language.parse_response("400:p0,100:dr") + assert "p0" in r.spec() + s = r.preview_safe() + assert "p0" not in s.spec() + def test_raw(self): s = cStringIO.StringIO() r = language.parse_response("400:b'foo'") @@ -651,12 +656,6 @@ class TestResponse: r = language.parse_response("400:m'msg':b@100:d0:i0,'foo'") testlen(r) - def test_render(self): - r = language.parse_response("400:p0,100:dr") - assert "p0" in r.spec() - s = r.preview_safe() - assert not "p0" in s.spec() - def test_parse_err(self): tutils.raises(language.ParseException, language.parse_response, "400:msg,b:") try: @@ -685,9 +684,10 @@ class TestResponse: assert r.actions[0].spec() == "pr,10" def test_parse_stress(self): - # While larger values are known to work on linux, - # len() technically returns an int and a python 2.7 int on windows has 32bit precision. - # Therefore, we should keep the body length < 2147483647 bytes in our tests. + # While larger values are known to work on linux, len() technically + # returns an int and a python 2.7 int on windows has 32bit precision. + # Therefore, we should keep the body length < 2147483647 bytes in our + # tests. r = language.parse_response("400:b@1g") assert r.length({}) @@ -700,16 +700,29 @@ class TestResponse: rt("400:da") - def test_read_file(): tutils.raises(language.FileAccessDenied, language.read_file, {}, "=/foo") p = tutils.test_data.path("data") d = dict(staticdir=p) assert language.read_file(d, "+./file").strip() == "testfile" assert language.read_file(d, "+file").strip() == "testfile" - tutils.raises(language.FileAccessDenied, language.read_file, d, "+./nonexistent") - tutils.raises(language.FileAccessDenied, language.read_file, d, "+/nonexistent") - - tutils.raises(language.FileAccessDenied, language.read_file, d, "+../test_language.py") + tutils.raises( + language.FileAccessDenied, + language.read_file, + d, + "+./nonexistent" + ) + tutils.raises( + language.FileAccessDenied, + language.read_file, + d, + "+/nonexistent" + ) + tutils.raises( + language.FileAccessDenied, + language.read_file, + d, + "+../test_language.py" + ) d["unconstrained_file_access"] = True assert language.read_file(d, "+../test_language.py")