From 9fda74c65a632bda5176e2ccafbbcab2af27d77d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 4 Jun 2015 17:18:06 +1200 Subject: [PATCH] Clarify language API, stub out nested websocket frames --- libpathod/app.py | 4 +- libpathod/cmdline.py | 4 +- libpathod/language/__init__.py | 11 ++++-- libpathod/language/websockets.py | 42 +++++++++++++++----- libpathod/pathoc.py | 6 +-- libpathod/pathod.py | 2 +- test/test_language_actions.py | 20 +++++----- test/test_language_base.py | 4 +- test/test_language_http.py | 66 ++++++++++++++++---------------- test/test_language_websocket.py | 5 +-- test/test_language_writer.py | 6 +-- test/test_pathoc.py | 4 +- test/tutils.py | 2 +- 13 files changed, 100 insertions(+), 76 deletions(-) diff --git a/libpathod/app.py b/libpathod/app.py index 20225ff77..a90582799 100644 --- a/libpathod/app.py +++ b/libpathod/app.py @@ -135,9 +135,9 @@ def make_app(noapi, debug): try: if is_request: - r = language.parse_requests(spec)[0] + r = language.parse_pathoc(spec)[0] else: - r = language.parse_response(spec) + r = language.parse_pathod(spec) except language.ParseException as v: args["syntaxerror"] = str(v) args["marked"] = v.marked() diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py index 67d5646a3..2279262db 100644 --- a/libpathod/cmdline.py +++ b/libpathod/cmdline.py @@ -189,7 +189,7 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): data = open(r).read() r = data try: - reqs.extend(language.parse_requests(r)) + reqs.extend(language.parse_pathoc(r)) except language.ParseException as v: print >> stderr, "Error parsing request spec: %s" % v.msg print >> stderr, v.marked() @@ -400,7 +400,7 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): data = open(spec).read() spec = data try: - req = language.parse_response(spec) + req = language.parse_pathod(spec) except language.ParseException as v: print >> stderr, "Error parsing anchor spec: %s" % v.msg print >> stderr, v.marked() diff --git a/libpathod/language/__init__.py b/libpathod/language/__init__.py index 3cc7dfbe2..e2e3e57d7 100644 --- a/libpathod/language/__init__.py +++ b/libpathod/language/__init__.py @@ -9,7 +9,7 @@ from base import Settings assert Settings # prevent pyflakes from messing with this -def parse_response(s): +def parse_pathod(s): """ May raise ParseException """ @@ -18,12 +18,17 @@ def parse_response(s): except UnicodeError: raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0) try: - return http.Response.expr().parseString(s, parseAll=True)[0] + return pp.Or( + [ + websockets.WebsocketFrame.expr(), + http.Response.expr(), + ] + ).parseString(s, parseAll=True)[0] except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) -def parse_requests(s): +def parse_pathoc(s): """ May raise ParseException """ diff --git a/libpathod/language/websockets.py b/libpathod/language/websockets.py index 46daa467d..3869bd85a 100644 --- a/libpathod/language/websockets.py +++ b/libpathod/language/websockets.py @@ -3,16 +3,6 @@ import netlib.websockets import pyparsing as pp from . import base, generators, actions, message -""" - wf:ctext:b'foo' - wf:c15:r'foo' - wf:fin:rsv1:rsv2:rsv3:mask - wf:-fin:-rsv1:-rsv2:-rsv3:-mask - wf:l234 - - wf:mask:r"foo -""" - class WF(base.CaselessLiteral): TOK = "wf" @@ -79,6 +69,38 @@ class Times(base.Integer): preamble = "x" +class NestedFrame(base.Token): + def __init__(self, value): + self.value = value + try: + self.parsed = WebsocketFrame( + Response.expr().parseString( + value.val, + parseAll=True + ) + ) + except pp.ParseException as v: + raise exceptions.ParseException(v.msg, v.line, v.col) + + @classmethod + def expr(klass): + e = pp.Literal("wf").suppress() + e = e + base.TokValueLiteral.expr() + return e.setParseAction(lambda x: klass(*x)) + + def values(self, settings): + return [ + self.value.get_generator(settings), + ] + + def spec(self): + return "s%s" % (self.value.spec()) + + def freeze(self, settings): + f = self.parsed.freeze(settings).spec() + return NestedFrame(base.TokValueLiteral(f.encode("string_escape"))) + + class WebsocketFrame(message.Message): comps = ( OpCode, diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index 59c88910c..f7268193c 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -270,7 +270,7 @@ class Pathoc(tcp.TCPClient): """ with self.log() as log: if isinstance(r, basestring): - r = language.parse_requests(r)[0] + r = language.parse_pathoc(r)[0] log(">> %s" % r) try: language.serve(r, self.wfile, self.settings) @@ -316,7 +316,7 @@ class Pathoc(tcp.TCPClient): """ with self.log() as log: if isinstance(r, basestring): - r = language.parse_requests(r)[0] + r = language.parse_pathoc(r)[0] log(">> %s" % r) resp, req = None, None try: @@ -355,7 +355,7 @@ class Pathoc(tcp.TCPClient): May raise http.HTTPError, tcp.NetLibError """ if isinstance(r, basestring): - r = language.parse_requests(r)[0] + r = language.parse_pathoc(r)[0] if isinstance(r, language.http.Request): if r.ws: return self.websocket_start(r, self.websocket_get_frame) diff --git a/libpathod/pathod.py b/libpathod/pathod.py index c5ad942ab..f536ce387 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -226,7 +226,7 @@ class PathodHandler(tcp.BaseHandler): spec = "ws" lg("crafting spec: %s" % spec) try: - crafted = language.parse_response(spec) + crafted = language.parse_pathod(spec) except language.ParseException as v: lg("Parse error: %s" % v.msg) crafted = language.http.make_error_response( diff --git a/test/test_language_actions.py b/test/test_language_actions.py index 16d54806f..b0d978871 100644 --- a/test/test_language_actions.py +++ b/test/test_language_actions.py @@ -5,7 +5,7 @@ from libpathod import language def parse_request(s): - return language.parse_requests(s)[0] + return language.parse_pathoc(s)[0] def test_unique_name(): @@ -14,10 +14,10 @@ def test_unique_name(): class TestDisconnects: - def test_parse_response(self): - a = language.parse_response("400:d0").actions[0] + def test_parse_pathod(self): + a = language.parse_pathod("400:d0").actions[0] assert a.spec() == "d0" - a = language.parse_response("400:dr").actions[0] + a = language.parse_pathod("400:dr").actions[0] assert a.spec() == "dr" def test_at(self): @@ -39,13 +39,13 @@ class TestDisconnects: class TestInject: - def test_parse_response(self): - a = language.parse_response("400:ir,@100").actions[0] + def test_parse_pathod(self): + a = language.parse_pathod("400:ir,@100").actions[0] assert a.offset == "r" assert a.value.datatype == "bytes" assert a.value.usize == 100 - a = language.parse_response("400:ia,@100").actions[0] + a = language.parse_pathod("400:ia,@100").actions[0] assert a.offset == "a" def test_at(self): @@ -60,7 +60,7 @@ class TestInject: def test_serve(self): s = cStringIO.StringIO() - r = language.parse_response("400:i0,'foo'") + r = language.parse_pathod("400:i0,'foo'") assert language.serve(r, s, {}) def test_spec(self): @@ -77,7 +77,7 @@ class TestInject: class TestPauses: - def test_parse_response(self): + def test_parse_pathod(self): e = actions.PauseAt.expr() v = e.parseString("p10,10")[0] assert v.seconds == 10 @@ -93,7 +93,7 @@ class TestPauses: assert v.offset == "a" def test_request(self): - r = language.parse_response('400:p10,10') + r = language.parse_pathod('400:p10,10') assert r.actions[0].spec() == "p10,10" def test_spec(self): diff --git a/test/test_language_base.py b/test/test_language_base.py index 020d68edb..e14b741fd 100644 --- a/test/test_language_base.py +++ b/test/test_language_base.py @@ -6,11 +6,11 @@ import nose.tools as nt def parse_request(s): - return language.parse_requests(s)[0] + return language.parse_pathoc(s)[0] def test_times(): - reqs = language.parse_requests("get:/:x5") + reqs = language.parse_pathoc("get:/:x5") assert len(reqs) == 5 assert not reqs[0].times diff --git a/test/test_language_http.py b/test/test_language_http.py index 3bdd0ec57..4851beaa2 100644 --- a/test/test_language_http.py +++ b/test/test_language_http.py @@ -6,7 +6,7 @@ import tutils def parse_request(s): - return language.parse_requests(s)[0] + return language.parse_pathoc(s)[0] def test_make_error_response(): @@ -32,7 +32,7 @@ class TestRequest: assert len(r.path.string()) == 1024 def test_multiple(self): - r = language.parse_requests("GET:/ PUT:/") + r = language.parse_pathoc("GET:/ PUT:/") assert r[0].method.string() == "GET" assert r[1].method.string() == "PUT" assert len(r) == 2 @@ -52,7 +52,7 @@ class TestRequest: ir,@1 """ - r = language.parse_requests(l) + r = language.parse_pathoc(l) assert len(r) == 2 assert r[0].method.string() == "GET" assert r[1].method.string() == "PUT" @@ -61,14 +61,14 @@ class TestRequest: get:"http://localhost:9999/p/200":ir,@1 get:"http://localhost:9999/p/200":ir,@2 """ - r = language.parse_requests(l) + r = language.parse_pathoc(l) assert len(r) == 2 assert r[0].method.string() == "GET" assert r[1].method.string() == "GET" def test_pathodspec(self): l = "get:/p:s'200'" - r = language.parse_requests(l) + r = language.parse_pathoc(l) assert len(r) == 1 assert len(r[0].tokens) == 3 assert isinstance(r[0].tokens[2], http.PathodResponse) @@ -143,42 +143,42 @@ class TestRequest: class TestResponse: def dummy_response(self): - return language.parse_response("400'msg'") + return language.parse_pathod("400'msg'") def test_response(self): - r = language.parse_response("400:m'msg'") + r = language.parse_pathod("400:m'msg'") assert r.code.string() == "400" assert r.reason.string() == "msg" - r = language.parse_response("400:m'msg':b@100b") + r = language.parse_pathod("400:m'msg':b@100b") assert r.reason.string() == "msg" assert r.body.values({}) assert str(r) - r = language.parse_response("200") + r = language.parse_pathod("200") assert r.code.string() == "200" assert not r.reason assert "OK" in [i[:] for i in r.preamble({})] def test_render(self): s = cStringIO.StringIO() - r = language.parse_response("400:m'msg'") + r = language.parse_pathod("400:m'msg'") assert language.serve(r, s, {}) - r = language.parse_response("400:p0,100:dr") + r = language.parse_pathod("400:p0,100:dr") assert "p0" in r.spec() s = r.preview_safe() assert "p0" not in s.spec() def test_raw(self): s = cStringIO.StringIO() - r = language.parse_response("400:b'foo'") + r = language.parse_pathod("400:b'foo'") language.serve(r, s, {}) v = s.getvalue() assert "Content-Length" in v s = cStringIO.StringIO() - r = language.parse_response("400:b'foo':r") + r = language.parse_pathod("400:b'foo':r") language.serve(r, s, {}) v = s.getvalue() assert "Content-Length" not in v @@ -188,9 +188,9 @@ class TestResponse: s = cStringIO.StringIO() language.serve(x, s, language.Settings()) assert x.length(language.Settings()) == len(s.getvalue()) - testlen(language.parse_response("400:m'msg':r")) - testlen(language.parse_response("400:m'msg':h'foo'='bar':r")) - testlen(language.parse_response("400:m'msg':h'foo'='bar':b@100b:r")) + testlen(language.parse_pathod("400:m'msg':r")) + testlen(language.parse_pathod("400:m'msg':h'foo'='bar':r")) + testlen(language.parse_pathod("400:m'msg':h'foo'='bar':b@100b:r")) def test_maximum_length(self): def testlen(x): @@ -199,42 +199,42 @@ class TestResponse: language.serve(x, s, {}) assert m >= len(s.getvalue()) - r = language.parse_response("400:m'msg':b@100:d0") + r = language.parse_pathod("400:m'msg':b@100:d0") testlen(r) - r = language.parse_response("400:m'msg':b@100:d0:i0,'foo'") + r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'") testlen(r) - r = language.parse_response("400:m'msg':b@100:d0:i0,'foo'") + r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'") testlen(r) def test_parse_err(self): tutils.raises( - language.ParseException, language.parse_response, "400:msg,b:" + language.ParseException, language.parse_pathod, "400:msg,b:" ) try: - language.parse_response("400'msg':b:") + language.parse_pathod("400'msg':b:") except language.ParseException as v: assert v.marked() assert str(v) def test_nonascii(self): - tutils.raises("ascii", language.parse_response, "foo:b\xf0") + tutils.raises("ascii", language.parse_pathod, "foo:b\xf0") def test_parse_header(self): - r = language.parse_response('400:h"foo"="bar"') + r = language.parse_pathod('400:h"foo"="bar"') assert http.get_header("foo", r.headers) def test_parse_pause_before(self): - r = language.parse_response("400:p0,10") + r = language.parse_pathod("400:p0,10") assert r.actions[0].spec() == "p0,10" def test_parse_pause_after(self): - r = language.parse_response("400:pa,10") + r = language.parse_pathod("400:pa,10") assert r.actions[0].spec() == "pa,10" def test_parse_pause_random(self): - r = language.parse_response("400:pr,10") + r = language.parse_pathod("400:pr,10") assert r.actions[0].spec() == "pr,10" def test_parse_stress(self): @@ -242,19 +242,19 @@ class TestResponse: # returns an int and a python 2.7 int on windows has 32bit precision. # Therefore, we should keep the body length < 2147483647 bytes in our # tests. - r = language.parse_response("400:b@1g") + r = language.parse_pathod("400:b@1g") assert r.length({}) def test_spec(self): def rt(s): - s = language.parse_response(s).spec() - assert language.parse_response(s).spec() == s + s = language.parse_pathod(s).spec() + assert language.parse_pathod(s).spec() == s rt("400:b@100g") rt("400") rt("400:da") def test_websockets(self): - r = language.parse_response("ws") + r = language.parse_pathod("ws") tutils.raises("no websocket key", r.resolve, language.Settings()) res = r.resolve(language.Settings(websocket_key="foo")) assert res.code.string() == "101" @@ -293,9 +293,9 @@ def test_location_shortcut(): def test_shortcuts(): - assert language.parse_response( + assert language.parse_pathod( "400:c'foo'").headers[0].key.val == "Content-Type" - assert language.parse_response( + assert language.parse_pathod( "400:l'foo'").headers[0].key.val == "Location" assert "Android" in tutils.render(parse_request("get:/:ua")) @@ -349,6 +349,6 @@ def test_pathodspec_freeze(): def test_unique_components(): tutils.raises( "multiple body clauses", - language.parse_response, + language.parse_pathod, "400:b@1:b@1" ) diff --git a/test/test_language_websocket.py b/test/test_language_websocket.py index faf9299fb..f55e6e37c 100644 --- a/test/test_language_websocket.py +++ b/test/test_language_websocket.py @@ -6,7 +6,7 @@ import tutils def parse_request(s): - return language.parse_requests(s)[0] + return language.parse_pathoc(s)[0] class TestWebsocketFrame: @@ -36,9 +36,6 @@ class TestWebsocketFrame: wf2 = parse_request(spec) assert wf2.spec() == spec - def test_raw(self): - pass - def test_flags(self): wf = parse_request("wf:fin:mask:rsv1:rsv2:rsv3") frm = netlib.websockets.Frame.from_bytes(tutils.render(wf)) diff --git a/test/test_language_writer.py b/test/test_language_writer.py index 406655d9a..7771176ba 100644 --- a/test/test_language_writer.py +++ b/test/test_language_writer.py @@ -78,14 +78,14 @@ def test_write_values_pauses(): def test_write_values_after(): s = cStringIO.StringIO() - r = language.parse_response("400:da") + r = language.parse_pathod("400:da") language.serve(r, s, {}) s = cStringIO.StringIO() - r = language.parse_response("400:pa,0") + r = language.parse_pathod("400:pa,0") language.serve(r, s, {}) s = cStringIO.StringIO() - r = language.parse_response("400:ia,'xx'") + r = language.parse_pathod("400:ia,'xx'") language.serve(r, s, {}) assert s.getvalue().endswith('xx') diff --git a/test/test_pathoc.py b/test/test_pathoc.py index 1735f084a..05519e4d4 100644 --- a/test/test_pathoc.py +++ b/test/test_pathoc.py @@ -22,7 +22,7 @@ class _TestDaemon: ssloptions = self.ssloptions, staticdir = tutils.test_data.path("data"), anchors = [ - (re.compile("/anchor/.*"), language.parse_response("202")) + (re.compile("/anchor/.*"), language.parse_pathod("202")) ] ) @@ -73,7 +73,7 @@ class _TestDaemon: if timeout: c.settimeout(timeout) for i in requests: - r = language.parse_requests(i)[0] + r = language.parse_pathoc(i)[0] if explain: r = r.freeze(language.Settings()) try: diff --git a/test/tutils.py b/test/tutils.py index 842ed527a..778ad7909 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -26,7 +26,7 @@ class DaemonTests(object): klass.d = test.Daemon( staticdir=test_data.path("data"), anchors=[ - (re.compile("/anchor/.*"), language.parse_response("202:da")) + (re.compile("/anchor/.*"), language.parse_pathod("202:da")) ], ssl = klass.ssl, ssloptions = so,