clean up flow reading

libmproxy/console/__init__.py

@@ -499,38 +499,33 @@ class ConsoleMaster(flow.FlowMaster):
         self.view_flowlist()
 
     def _readflow(self, paths):
+        """
+        Utitility function that reads a list of flows
+        or prints an error to the UI if that fails.
+        Returns
+            - None, if there was an error.
+            - a list of flows, otherwise.
+        """
         try:
-            flows = []
+            return flow.read_flows_from_paths(paths)
-            for path in paths:
+        except flow.FlowReadError as e:
-                path = os.path.expanduser(path)
+            if not self.statusbar:
-                with file(path, "rb") as f:
+                print >> sys.stderr, e.strerror
-                    flows.extend(list(flow.FlowReader(f).stream()))
+                sys.exit(1)
-        except (IOError, flow.FlowReadError), v:
+            else:
-            return True, v.strerror
+                self.statusbar.message(e.strerror)
-        return False, flows
+            return None
 
     def client_playback_path(self, path):
-        err, ret = self._readflow(path)
+        flows = self._readflow(path)
-        if err:
+        if flows:
-            if not self.statusbar:
+            self.start_client_playback(flows, False)
-                print >> sys.stderr, ret
-                sys.exit(1)
-            else:
-                self.statusbar.message(ret)
-        else:
-            self.start_client_playback(ret, False)
 
     def server_playback_path(self, path):
-        err, ret = self._readflow(path)
+        flows = self._readflow(path)
-        if err:
+        if flows:
-            if not self.statusbar:
-                print >> sys.stderr, ret
-                sys.exit(1)
-            else:
-                self.statusbar.message(ret)
-        else:
             self.start_server_playback(
-                ret,
+                flows,
                 self.killextra, self.rheaders,
                 False, self.nopop,
                 self.options.replay_ignore_params, self.options.replay_ignore_content, self.options.replay_ignore_payload_params

libmproxy/dump.py

@@ -144,15 +144,14 @@ class DumpMaster(flow.FlowMaster):
             self.start_app(self.o.app_host, self.o.app_port)
 
     def _readflow(self, paths):
+        """
+        Utitility function that reads a list of flows
+        or raises a DumpError if that fails.
+        """
         try:
-            flows = []
+            return flow.read_flows_from_paths(paths)
-            for path in paths:
+        except flow.FlowReadError as e:
-                path = os.path.expanduser(path)
+            raise DumpError(e.strerror)
-                with file(path, "rb") as f:
-                    flows.extend(list(flow.FlowReader(f).stream()))
-        except (IOError, flow.FlowReadError), v:
-            raise DumpError(v.strerror)
-        return flows
 
     def add_event(self, e, level="info"):
         needed = dict(error=0, info=1, debug=2).get(level, 1)

libmproxy/flow.py

@@ -945,6 +945,25 @@ class FlowMaster(controller.Master):
         self.stream = None
 
 
+def read_flows_from_paths(paths):
+    """
+    Given a list of filepaths, read all flows and return a list of them.
+    From a performance perspective, streaming would be advisable -
+    however, if there's an error with one of the files, we want it to be raised immediately.
+
+    If an error occurs, a FlowReadError will be raised.
+    """
+    try:
+        flows = []
+        for path in paths:
+            path = os.path.expanduser(path)
+            with file(path, "rb") as f:
+                flows.extend(FlowReader(f).stream())
+    except IOError as e:
+        raise FlowReadError(e.strerror)
+    return flows
+
+
 class FlowWriter:
     def __init__(self, fo):
         self.fo = fo

test/test_dump.py

@@ -75,7 +75,7 @@ class TestDumpMaster:
     def test_replay(self):
         cs = StringIO()
 
-        o = dump.Options(server_replay="nonexistent", kill=True)
+        o = dump.Options(server_replay=["nonexistent"], kill=True)
         tutils.raises(dump.DumpError, dump.DumpMaster, None, o, outfile=cs)
 
         with tutils.tmpdir() as t: