From 0da3e51e1c08eed2c8054d4a71bc591268b19af1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 7 Jun 2015 13:18:33 +1200 Subject: [PATCH] Make parse_pathoc a generator This lets us do things like this: get:/:ir,@1:x1000000000 It will also let us expand the language to include a "repeat forever" concept. --- libpathod/app.py | 2 +- libpathod/cmdline.py | 3 ++- libpathod/language/__init__.py | 18 +++++++------ libpathod/pathoc.py | 45 +++++++++++++++------------------ libpathod/pathod.py | 4 +-- test/test_cmdline.py | 2 +- test/test_language_actions.py | 2 +- test/test_language_base.py | 4 +-- test/test_language_http.py | 10 ++++---- test/test_language_websocket.py | 2 +- test/test_pathoc.py | 2 +- 11 files changed, 46 insertions(+), 48 deletions(-) diff --git a/libpathod/app.py b/libpathod/app.py index a90582799..cb4ad5aac 100644 --- a/libpathod/app.py +++ b/libpathod/app.py @@ -135,7 +135,7 @@ def make_app(noapi, debug): try: if is_request: - r = language.parse_pathoc(spec)[0] + r = language.parse_pathoc(spec).next() else: r = language.parse_pathod(spec) except language.ParseException as v: diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py index ac03a5d49..d89608a2e 100644 --- a/libpathod/cmdline.py +++ b/libpathod/cmdline.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import itertools import argparse import os import os.path @@ -189,7 +190,7 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): data = open(r).read() r = data try: - reqs.extend(language.parse_pathoc(r)) + reqs.append(language.parse_pathoc(r)) except language.ParseException as v: print >> stderr, "Error parsing request spec: %s" % v.msg print >> stderr, v.marked() diff --git a/libpathod/language/__init__.py b/libpathod/language/__init__.py index 48466fa64..d1ace600b 100644 --- a/libpathod/language/__init__.py +++ b/libpathod/language/__init__.py @@ -1,3 +1,4 @@ +import itertools import time import pyparsing as pp @@ -28,6 +29,14 @@ def parse_pathod(s): raise exceptions.ParseException(v.msg, v.line, v.col) +def expand(req): + if req.times: + for j in xrange(int(req.times.value)): + yield req.strike_token("times") + else: + yield req + + def parse_pathoc(s): """ May raise ParseException @@ -47,14 +56,7 @@ def parse_pathoc(s): ).parseString(s, parseAll=True) except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) - expanded = [] - for i in reqs: - if i.times: - for j in range(int(i.times.value)): - expanded.append(i.strike_token("times")) - else: - expanded.append(i) - return expanded + return itertools.chain(*[expand(i) for i in reqs]) def parse_websocket_frame(s): diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index 385b61db2..37e921e4a 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -1,5 +1,6 @@ import sys import os +import itertools import hashlib import Queue import random @@ -287,7 +288,7 @@ class Pathoc(tcp.TCPClient): """ with self.log() as log: if isinstance(r, basestring): - r = language.parse_pathoc(r)[0] + r = language.parse_pathoc(r).next() log(">> %s" % r) try: language.serve(r, self.wfile, self.settings) @@ -330,7 +331,7 @@ class Pathoc(tcp.TCPClient): """ with self.log() as log: if isinstance(r, basestring): - r = language.parse_pathoc(r)[0] + r = language.parse_pathoc(r).next() log(">> %s" % r) resp, req = None, None try: @@ -369,7 +370,7 @@ class Pathoc(tcp.TCPClient): May raise http.HTTPError, tcp.NetLibError """ if isinstance(r, basestring): - r = language.parse_pathoc(r)[0] + r = language.parse_pathoc(r).next() if isinstance(r, language.http.Request): if r.ws: return self.websocket_start(r, self.websocket_get_frame) @@ -388,17 +389,13 @@ def main(args): # pragma: nocover while True: if cnt == args.repeat and args.repeat != 0: break - if trycount > args.memolimit: - print >> sys.stderr, "Memo limit exceeded..." - return if args.wait and cnt != 0: time.sleep(args.wait) cnt += 1 + playlist = itertools.chain(*args.requests) if args.random: - playlist = [random.choice(args.requests)] - else: - playlist = args.requests + playlist = random.choice(args.requests) p = Pathoc( (args.host, args.port), ssl = args.ssl, @@ -414,22 +411,6 @@ def main(args): # pragma: nocover ignoretimeout = args.ignoretimeout, showsummary = True ) - if args.explain or args.memo: - playlist = [ - i.freeze(p.settings) for i in playlist - ] - if args.memo: - newlist = [] - for spec in playlist: - h = hashlib.sha256(spec.spec()).digest() - if h not in memo: - memo.add(h) - newlist.append(spec) - playlist = newlist - if not playlist: - trycount += 1 - continue - trycount = 0 try: p.connect(args.connect_to, args.showssl) @@ -442,6 +423,20 @@ def main(args): # pragma: nocover if args.timeout: p.settimeout(args.timeout) for spec in playlist: + if args.explain or args.memo: + spec = spec.freeze(p.settings) + if args.memo: + h = hashlib.sha256(spec.spec()).digest() + if h not in memo: + trycount = 0 + memo.add(h) + else: + trycount += 1 + if trycount > args.memolimit: + print >> sys.stderr, "Memo limit exceeded..." + return + else: + continue try: ret = p.request(spec) if ret and args.oneshot: diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 3cd631622..367a3163a 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -75,7 +75,7 @@ class PathodHandler(tcp.BaseHandler): def handle_sni(self, connection): self.sni = connection.get_servername() - def serve_crafted(self, crafted): + def http_serve_crafted(self, crafted): error, crafted = self.server.check_policy( crafted, self.settings ) @@ -304,7 +304,7 @@ class PathodHandler(tcp.BaseHandler): if anchor_spec: lg("crafting spec: %s" % anchor_spec) - nexthandler, retlog["response"] = self.serve_crafted( + nexthandler, retlog["response"] = self.http_serve_crafted( anchor_spec ) if nexthandler and websocket_key: diff --git a/test/test_cmdline.py b/test/test_cmdline.py index b9607029b..c1b556082 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -135,7 +135,7 @@ def test_pathoc(perror): tutils.test_data.path("data/request") ] ) - assert len(a.requests) == 1 + assert len(list(a.requests)) == 1 a = cmdline.args_pathod( [ diff --git a/test/test_language_actions.py b/test/test_language_actions.py index b0d978871..3ead141ac 100644 --- a/test/test_language_actions.py +++ b/test/test_language_actions.py @@ -5,7 +5,7 @@ from libpathod import language def parse_request(s): - return language.parse_pathoc(s)[0] + return language.parse_pathoc(s).next() def test_unique_name(): diff --git a/test/test_language_base.py b/test/test_language_base.py index e14b741fd..b2296e3aa 100644 --- a/test/test_language_base.py +++ b/test/test_language_base.py @@ -6,11 +6,11 @@ import nose.tools as nt def parse_request(s): - return language.parse_pathoc(s)[0] + return language.parse_pathoc(s).next() def test_times(): - reqs = language.parse_pathoc("get:/:x5") + reqs = list(language.parse_pathoc("get:/:x5")) assert len(reqs) == 5 assert not reqs[0].times diff --git a/test/test_language_http.py b/test/test_language_http.py index f1484da58..f9bbd2732 100644 --- a/test/test_language_http.py +++ b/test/test_language_http.py @@ -6,7 +6,7 @@ import tutils def parse_request(s): - return language.parse_pathoc(s)[0] + return language.parse_pathoc(s).next() def test_make_error_response(): @@ -32,7 +32,7 @@ class TestRequest: assert len(r.path.string()) == 1024 def test_multiple(self): - r = language.parse_pathoc("GET:/ PUT:/") + r = list(language.parse_pathoc("GET:/ PUT:/")) assert r[0].method.string() == "GET" assert r[1].method.string() == "PUT" assert len(r) == 2 @@ -52,7 +52,7 @@ class TestRequest: ir,@1 """ - r = language.parse_pathoc(l) + r = list(language.parse_pathoc(l)) assert len(r) == 2 assert r[0].method.string() == "GET" assert r[1].method.string() == "PUT" @@ -61,14 +61,14 @@ class TestRequest: get:"http://localhost:9999/p/200":ir,@1 get:"http://localhost:9999/p/200":ir,@2 """ - r = language.parse_pathoc(l) + r = list(language.parse_pathoc(l)) assert len(r) == 2 assert r[0].method.string() == "GET" assert r[1].method.string() == "GET" def test_nested_response(self): l = "get:/p:s'200'" - r = language.parse_pathoc(l) + r = list(language.parse_pathoc(l)) assert len(r) == 1 assert len(r[0].tokens) == 3 assert isinstance(r[0].tokens[2], http.NestedResponse) diff --git a/test/test_language_websocket.py b/test/test_language_websocket.py index f75562c9c..e7f333b44 100644 --- a/test/test_language_websocket.py +++ b/test/test_language_websocket.py @@ -6,7 +6,7 @@ import tutils def parse_request(s): - return language.parse_pathoc(s)[0] + return language.parse_pathoc(s).next() class TestWebsocketFrame: diff --git a/test/test_pathoc.py b/test/test_pathoc.py index 994975fce..556aada51 100644 --- a/test/test_pathoc.py +++ b/test/test_pathoc.py @@ -73,7 +73,7 @@ class _TestDaemon: if timeout: c.settimeout(timeout) for i in requests: - r = language.parse_pathoc(i)[0] + r = language.parse_pathoc(i).next() if explain: r = r.freeze(language.Settings()) try: