Make parse_pathoc a generator

This lets us do things like this:

get:/:ir,@1:x1000000000

It will also let us expand the language to include a "repeat forever" concept.

libpathod/app.py

@@ -135,7 +135,7 @@ def make_app(noapi, debug):
 
         try:
             if is_request:
-                r = language.parse_pathoc(spec)[0]
+                r = language.parse_pathoc(spec).next()
             else:
                 r = language.parse_pathod(spec)
         except language.ParseException as v:

libpathod/cmdline.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import itertools
 import argparse
 import os
 import os.path
@@ -189,7 +190,7 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
             data = open(r).read()
             r = data
         try:
-            reqs.extend(language.parse_pathoc(r))
+            reqs.append(language.parse_pathoc(r))
         except language.ParseException as v:
             print >> stderr, "Error parsing request spec: %s" % v.msg
             print >> stderr, v.marked()

libpathod/language/__init__.py

@@ -1,3 +1,4 @@
+import itertools
 import time
 
 import pyparsing as pp
@@ -28,6 +29,14 @@ def parse_pathod(s):
         raise exceptions.ParseException(v.msg, v.line, v.col)
 
 
+def expand(req):
+    if req.times:
+        for j in xrange(int(req.times.value)):
+            yield req.strike_token("times")
+    else:
+        yield req
+
+
 def parse_pathoc(s):
     """
         May raise ParseException
@@ -47,14 +56,7 @@ def parse_pathoc(s):
         ).parseString(s, parseAll=True)
     except pp.ParseException as v:
         raise exceptions.ParseException(v.msg, v.line, v.col)
-    expanded = []
+    return itertools.chain(*[expand(i) for i in reqs])
-    for i in reqs:
-        if i.times:
-            for j in range(int(i.times.value)):
-                expanded.append(i.strike_token("times"))
-        else:
-            expanded.append(i)
-    return expanded
 
 
 def parse_websocket_frame(s):

libpathod/pathoc.py

@@ -1,5 +1,6 @@
 import sys
 import os
+import itertools
 import hashlib
 import Queue
 import random
@@ -287,7 +288,7 @@ class Pathoc(tcp.TCPClient):
         """
         with self.log() as log:
             if isinstance(r, basestring):
-                r = language.parse_pathoc(r)[0]
+                r = language.parse_pathoc(r).next()
             log(">> %s" % r)
             try:
                 language.serve(r, self.wfile, self.settings)
@@ -330,7 +331,7 @@ class Pathoc(tcp.TCPClient):
         """
         with self.log() as log:
             if isinstance(r, basestring):
-                r = language.parse_pathoc(r)[0]
+                r = language.parse_pathoc(r).next()
             log(">> %s" % r)
             resp, req = None, None
             try:
@@ -369,7 +370,7 @@ class Pathoc(tcp.TCPClient):
             May raise http.HTTPError, tcp.NetLibError
         """
         if isinstance(r, basestring):
-            r = language.parse_pathoc(r)[0]
+            r = language.parse_pathoc(r).next()
         if isinstance(r, language.http.Request):
             if r.ws:
                 return self.websocket_start(r, self.websocket_get_frame)
@@ -388,17 +389,13 @@ def main(args):  # pragma: nocover
         while True:
             if cnt == args.repeat and args.repeat != 0:
                 break
-            if trycount > args.memolimit:
-                print >> sys.stderr, "Memo limit exceeded..."
-                return
             if args.wait and cnt != 0:
                 time.sleep(args.wait)
 
             cnt += 1
+            playlist = itertools.chain(*args.requests)
             if args.random:
-                playlist = [random.choice(args.requests)]
+                playlist = random.choice(args.requests)
-            else:
-                playlist = args.requests
             p = Pathoc(
                 (args.host, args.port),
                 ssl = args.ssl,
@@ -414,22 +411,6 @@ def main(args):  # pragma: nocover
                 ignoretimeout = args.ignoretimeout,
                 showsummary = True
             )
-            if args.explain or args.memo:
-                playlist = [
-                    i.freeze(p.settings) for i in playlist
-                ]
-            if args.memo:
-                newlist = []
-                for spec in playlist:
-                    h = hashlib.sha256(spec.spec()).digest()
-                    if h not in memo:
-                        memo.add(h)
-                        newlist.append(spec)
-                playlist = newlist
-            if not playlist:
-                trycount += 1
-                continue
-
             trycount = 0
             try:
                 p.connect(args.connect_to, args.showssl)
@@ -442,6 +423,20 @@ def main(args):  # pragma: nocover
             if args.timeout:
                 p.settimeout(args.timeout)
             for spec in playlist:
+                if args.explain or args.memo:
+                    spec = spec.freeze(p.settings)
+                if args.memo:
+                    h = hashlib.sha256(spec.spec()).digest()
+                    if h not in memo:
+                        trycount = 0
+                        memo.add(h)
+                    else:
+                        trycount += 1
+                        if trycount > args.memolimit:
+                            print >> sys.stderr, "Memo limit exceeded..."
+                            return
+                        else:
+                            continue
                 try:
                     ret = p.request(spec)
                     if ret and args.oneshot:

libpathod/pathod.py

@@ -75,7 +75,7 @@ class PathodHandler(tcp.BaseHandler):
     def handle_sni(self, connection):
         self.sni = connection.get_servername()
 
-    def serve_crafted(self, crafted):
+    def http_serve_crafted(self, crafted):
         error, crafted = self.server.check_policy(
             crafted, self.settings
         )
@@ -304,7 +304,7 @@ class PathodHandler(tcp.BaseHandler):
 
             if anchor_spec:
                 lg("crafting spec: %s" % anchor_spec)
-                nexthandler, retlog["response"] = self.serve_crafted(
+                nexthandler, retlog["response"] = self.http_serve_crafted(
                     anchor_spec
                 )
                 if nexthandler and websocket_key:

test/test_cmdline.py

@@ -135,7 +135,7 @@ def test_pathoc(perror):
             tutils.test_data.path("data/request")
         ]
     )
-    assert len(a.requests) == 1
+    assert len(list(a.requests)) == 1
 
     a = cmdline.args_pathod(
         [

test/test_language_actions.py

@@ -5,7 +5,7 @@ from libpathod import language
 
 
 def parse_request(s):
-    return language.parse_pathoc(s)[0]
+    return language.parse_pathoc(s).next()
 
 
 def test_unique_name():

test/test_language_base.py

@@ -6,11 +6,11 @@ import nose.tools as nt
 
 
 def parse_request(s):
-    return language.parse_pathoc(s)[0]
+    return language.parse_pathoc(s).next()
 
 
 def test_times():
-    reqs = language.parse_pathoc("get:/:x5")
+    reqs = list(language.parse_pathoc("get:/:x5"))
     assert len(reqs) == 5
     assert not reqs[0].times
 

test/test_language_http.py

@@ -6,7 +6,7 @@ import tutils
 
 
 def parse_request(s):
-    return language.parse_pathoc(s)[0]
+    return language.parse_pathoc(s).next()
 
 
 def test_make_error_response():
@@ -32,7 +32,7 @@ class TestRequest:
         assert len(r.path.string()) == 1024
 
     def test_multiple(self):
-        r = language.parse_pathoc("GET:/ PUT:/")
+        r = list(language.parse_pathoc("GET:/ PUT:/"))
         assert r[0].method.string() == "GET"
         assert r[1].method.string() == "PUT"
         assert len(r) == 2
@@ -52,7 +52,7 @@ class TestRequest:
 
             ir,@1
         """
-        r = language.parse_pathoc(l)
+        r = list(language.parse_pathoc(l))
         assert len(r) == 2
         assert r[0].method.string() == "GET"
         assert r[1].method.string() == "PUT"
@@ -61,14 +61,14 @@ class TestRequest:
             get:"http://localhost:9999/p/200":ir,@1
             get:"http://localhost:9999/p/200":ir,@2
         """
-        r = language.parse_pathoc(l)
+        r = list(language.parse_pathoc(l))
         assert len(r) == 2
         assert r[0].method.string() == "GET"
         assert r[1].method.string() == "GET"
 
     def test_nested_response(self):
         l = "get:/p:s'200'"
-        r = language.parse_pathoc(l)
+        r = list(language.parse_pathoc(l))
         assert len(r) == 1
         assert len(r[0].tokens) == 3
         assert isinstance(r[0].tokens[2], http.NestedResponse)

test/test_language_websocket.py

@@ -6,7 +6,7 @@ import tutils
 
 
 def parse_request(s):
-    return language.parse_pathoc(s)[0]
+    return language.parse_pathoc(s).next()
 
 
 class TestWebsocketFrame:

test/test_pathoc.py

@@ -73,7 +73,7 @@ class _TestDaemon:
         if timeout:
             c.settimeout(timeout)
         for i in requests:
-            r = language.parse_pathoc(i)[0]
+            r = language.parse_pathoc(i).next()
             if explain:
                 r = r.freeze(language.Settings())
             try: