Remove flow module entirely, move contents to top level

mitmproxy.flow.io -> mitmproxy.io
mitmproxy.flow.export -> mitmproxy.export

examples/flowbasic

@@ -8,7 +8,7 @@
     Note that request and response messages are not automatically replied to,
     so we need to implement handlers to do this.
 """
-from mitmproxy import flow, controller, options
+from mitmproxy import controller, options, master
 from mitmproxy.proxy import ProxyServer, ProxyConfig
 
 
@@ -37,7 +37,6 @@ class MyMaster(master.Master):
 
 opts = options.Options(cadir="~/.mitmproxy/")
 config = ProxyConfig(opts)
-state = state.State()
 server = ProxyServer(config)
-m = MyMaster(opts, server, state)
+m = MyMaster(opts, server)
 m.run()

examples/flowwriter.py

@@ -1,7 +1,6 @@
 import random
 import sys
-
+from mimtproxy import io
-from mitmproxy.flow import FlowWriter
 
 
 class Writer:
@@ -10,7 +9,7 @@ class Writer:
             f = sys.stdout
         else:
             f = open(path, "wb")
-        self.w = FlowWriter(f)
+        self.w = io.FlowWriter(f)
 
     def response(self, flow):
         if random.choice([True, False]):

examples/read_dumpfile

@@ -9,7 +9,7 @@ import pprint
 import sys
 
 with open(sys.argv[1], "rb") as logfile:
-    freader = flow.FlowReader(logfile)
+    freader = io.FlowReader(logfile)
     pp = pprint.PrettyPrinter(indent=4)
     try:
         for f in freader.stream():

mitmproxy/addons/clientplayback.py

@@ -1,4 +1,7 @@
-from mitmproxy import exceptions, flow, ctx
+from mitmproxy import exceptions
+from mitmproxy import ctx
+from mitmproxy import io
+
 
 
 class ClientPlayback:
@@ -21,7 +24,7 @@ class ClientPlayback:
             if options.client_replay:
                 ctx.log.info(options.client_replay)
                 try:
-                    flows = flow.read_flows_from_paths(options.client_replay)
+                    flows = io.read_flows_from_paths(options.client_replay)
                 except exceptions.FlowReadException as e:
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)

mitmproxy/addons/filestreamer.py

@@ -2,7 +2,7 @@ import os.path
 
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
-from mitmproxy.flow import io
+from mitmproxy import io
 
 
 class FileStreamer:

mitmproxy/addons/serverplayback.py

@@ -2,7 +2,9 @@ import urllib
 import hashlib
 
 from netlib import strutils
-from mitmproxy import exceptions, flow, ctx
+from mitmproxy import exceptions
+from mitmproxy import ctx
+from mitmproxy import io
 
 
 class ServerPlayback:
@@ -91,7 +93,7 @@ class ServerPlayback:
             self.clear()
             if options.server_replay:
                 try:
-                    flows = flow.read_flows_from_paths(options.server_replay)
+                    flows = io.read_flows_from_paths(options.server_replay)
                 except exceptions.FlowReadException as e:
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)

mitmproxy/console/common.py

@@ -9,7 +9,7 @@ import urwid.util
 import netlib
 from mitmproxy import utils
 from mitmproxy.console import signals
-from mitmproxy.flow import export
+from mitmproxy import export
 from netlib import human
 
 try:

mitmproxy/console/flowlist.py

@@ -4,7 +4,7 @@ import netlib.http.url
 from mitmproxy import exceptions
 from mitmproxy.console import common
 from mitmproxy.console import signals
-from mitmproxy.flow import export
+from mitmproxy import export
 
 
 def _mkhelp():

mitmproxy/console/flowview.py

@@ -15,7 +15,7 @@ from mitmproxy.console import grideditor
 from mitmproxy.console import searchable
 from mitmproxy.console import signals
 from mitmproxy.console import tabs
-from mitmproxy.flow import export
+from mitmproxy import export
 from netlib.http import Headers
 from netlib.http import status_codes
 

mitmproxy/console/master.py

@@ -19,7 +19,7 @@ from mitmproxy import contentviews
 from mitmproxy import controller
 from mitmproxy import exceptions
 from mitmproxy import master
-from mitmproxy import flow
+from mitmproxy import io
 from mitmproxy import flowfilter
 from mitmproxy import utils
 from mitmproxy.addons import state
@@ -346,7 +346,7 @@ class ConsoleMaster(master.Master):
             - a list of flows, otherwise.
         """
         try:
-            return flow.read_flows_from_paths(path)
+            return io.read_flows_from_paths(path)
         except exceptions.FlowReadException as e:
             signals.status_message.send(message=str(e))
 
@@ -580,7 +580,7 @@ class ConsoleMaster(master.Master):
         path = os.path.expanduser(path)
         try:
             f = open(path, "wb")
-            fw = flow.FlowWriter(f)
+            fw = io.FlowWriter(f)
             for i in flows:
                 fw.add(i)
             f.close()

mitmproxy/dump.py

@@ -3,8 +3,8 @@ from typing import Optional
 
 from mitmproxy import controller
 from mitmproxy import exceptions
-from mitmproxy import flow
 from mitmproxy import addons
+from mitmproxy import io
 from mitmproxy import options
 from mitmproxy import master
 from mitmproxy.addons import dumper, termlog
@@ -68,7 +68,7 @@ class DumpMaster(master.Master):
         or raises a DumpError if that fails.
         """
         try:
-            return flow.read_flows_from_paths(paths)
+            return io.read_flows_from_paths(paths)
         except exceptions.FlowReadException as e:
             raise DumpError(str(e))
 

mitmproxy/flow/__init__.py

@@ -1,7 +0,0 @@
-from mitmproxy.flow import export
-from mitmproxy.flow.io import FlowWriter, FilteredFlowWriter, FlowReader, read_flows_from_paths
-
-__all__ = [
-    "export",
-    "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths",
-]

mitmproxy/io.py

@@ -4,7 +4,7 @@ from mitmproxy import exceptions
 from mitmproxy import flowfilter
 from mitmproxy import models
 from mitmproxy.contrib import tnetstring
-from mitmproxy.flow import io_compat
+from mitmproxy import io_compat
 
 
 class FlowWriter:

mitmproxy/master.py

@@ -11,7 +11,7 @@ from mitmproxy import events
 from mitmproxy import exceptions
 from mitmproxy import models
 from mitmproxy import log
-from mitmproxy.flow import io
+from mitmproxy import io
 from mitmproxy.protocol import http_replay
 from netlib import basethread
 from netlib import http

mitmproxy/web/app.py

@@ -9,11 +9,11 @@ import hashlib
 import tornado.websocket
 import tornado.web
 from io import BytesIO
-from mitmproxy.flow import FlowWriter, FlowReader
 
 from mitmproxy import flowfilter
 from mitmproxy import models
 from mitmproxy import contentviews
+from mitmproxy import io
 from netlib import version
 
 
@@ -193,7 +193,7 @@ class DumpFlows(RequestHandler):
         self.set_header("Content-Type", "application/octet-stream")
 
         bio = BytesIO()
-        fw = FlowWriter(bio)
+        fw = io.FlowWriter(bio)
         for f in self.state.flows:
             fw.add(f)
 
@@ -205,7 +205,7 @@ class DumpFlows(RequestHandler):
 
         content = self.request.files.values()[0][0].body
         bio = BytesIO(content)
-        self.state.load_flows(FlowReader(bio).stream())
+        self.state.load_flows(io.FlowReader(bio).stream())
         bio.close()
 
 

test/mitmproxy/addons/test_filestreamer.py

@@ -4,7 +4,7 @@ import os.path
 
 from mitmproxy.addons import filestreamer
 from mitmproxy import master
-from mitmproxy.flow import io
+from mitmproxy import io
 from mitmproxy import options
 from mitmproxy import proxy
 

test/mitmproxy/mastertest.py

@@ -4,7 +4,7 @@ from . import tutils
 import netlib.tutils
 
 from mitmproxy import master
-from mitmproxy import flow, proxy, models, options
+from mitmproxy import io, proxy, models, options
 
 
 class TestMaster:
@@ -33,7 +33,7 @@ class MasterTest:
 
     def flowfile(self, path):
         f = open(path, "wb")
-        fw = flow.FlowWriter(f)
+        fw = io.FlowWriter(f)
         t = tutils.tflow(resp=True)
         fw.add(t)
         f.close()

test/mitmproxy/test_dump.py

@@ -1,7 +1,8 @@
 import os
 import io
 
-from mitmproxy import dump, flow, exceptions, proxy
+import mitmproxy.io
+from mitmproxy import dump, exceptions, proxy
 from . import tutils, mastertest
 
 
@@ -127,7 +128,7 @@ class TestDumpMaster(mastertest.MasterTest):
             self.dummy_cycle(
                 self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, b""
             )
-            assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 1
+            assert len(list(mitmproxy.io.FlowReader(open(p, "rb")).stream())) == 1
 
     def test_write_append(self):
         with tutils.tmpdir() as d:
@@ -140,7 +141,7 @@ class TestDumpMaster(mastertest.MasterTest):
                 self.mkmaster(None, outfile=(p, "ab"), verbosity=0),
                 1, b""
             )
-            assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 2
+            assert len(list(mitmproxy.io.FlowReader(open(p, "rb")).stream())) == 2
 
     def test_write_err(self):
         tutils.raises(

test/mitmproxy/test_flow.py

@@ -3,7 +3,8 @@ import io
 
 import netlib.utils
 from netlib.http import Headers
-from mitmproxy import flowfilter, flow, options
+import mitmproxy.io
+from mitmproxy import flowfilter, options
 from mitmproxy.addons import state
 from mitmproxy.contrib import tnetstring
 from mitmproxy.exceptions import FlowReadException, Kill
@@ -329,7 +330,7 @@ class TestSerialize:
 
     def _treader(self):
         sio = io.BytesIO()
-        w = flow.FlowWriter(sio)
+        w = mitmproxy.io.FlowWriter(sio)
         for i in range(3):
             f = tutils.tflow(resp=True)
             w.add(f)
@@ -342,18 +343,18 @@ class TestSerialize:
         w.add(f)
 
         sio.seek(0)
-        return flow.FlowReader(sio)
+        return mitmproxy.io.FlowReader(sio)
 
     def test_roundtrip(self):
         sio = io.BytesIO()
         f = tutils.tflow()
         f.marked = True
         f.request.content = bytes(bytearray(range(256)))
-        w = flow.FlowWriter(sio)
+        w = mitmproxy.io.FlowWriter(sio)
         w.add(f)
 
         sio.seek(0)
-        r = flow.FlowReader(sio)
+        r = mitmproxy.io.FlowReader(sio)
         l = list(r.stream())
         assert len(l) == 1
 
@@ -386,7 +387,7 @@ class TestSerialize:
     def test_filter(self):
         sio = io.BytesIO()
         flt = flowfilter.parse("~c 200")
-        w = flow.FilteredFlowWriter(sio, flt)
+        w = mitmproxy.io.FilteredFlowWriter(sio, flt)
 
         f = tutils.tflow(resp=True)
         f.response.status_code = 200
@@ -397,14 +398,14 @@ class TestSerialize:
         w.add(f)
 
         sio.seek(0)
-        r = flow.FlowReader(sio)
+        r = mitmproxy.io.FlowReader(sio)
         assert len(list(r.stream()))
 
     def test_error(self):
         sio = io.BytesIO()
         sio.write(b"bogus")
         sio.seek(0)
-        r = flow.FlowReader(sio)
+        r = mitmproxy.io.FlowReader(sio)
         tutils.raises(FlowReadException, list, r.stream())
 
         f = FlowReadException("foo")
@@ -418,7 +419,7 @@ class TestSerialize:
         tnetstring.dump(d, sio)
         sio.seek(0)
 
-        r = flow.FlowReader(sio)
+        r = mitmproxy.io.FlowReader(sio)
         tutils.raises("version", list, r.stream())
 
 

test/mitmproxy/test_flow_export.py

@@ -2,7 +2,7 @@ import re
 
 import netlib.tutils
 from netlib.http import Headers
-from mitmproxy.flow import export  # heh
+from mitmproxy import export  # heh
 from . import tutils
 
 

test/mitmproxy/test_flow_format_compat.py

@@ -1,11 +1,11 @@
-from mitmproxy.flow import FlowReader
+from mitmproxy import io
-from mitmproxy.exceptions import FlowReadException
+from mitmproxy import exceptions
 from . import tutils
 
 
 def test_load():
     with open(tutils.test_data.path("data/dumpfile-011"), "rb") as f:
-        flow_reader = FlowReader(f)
+        flow_reader = io.FlowReader(f)
         flows = list(flow_reader.stream())
         assert len(flows) == 1
         assert flows[0].request.url == "https://example.com/"
@@ -13,6 +13,6 @@ def test_load():
 
 def test_cannot_convert():
     with open(tutils.test_data.path("data/dumpfile-010"), "rb") as f:
-        flow_reader = FlowReader(f)
+        flow_reader = io.FlowReader(f)
-        with tutils.raises(FlowReadException):
+        with tutils.raises(exceptions.FlowReadException):
             list(flow_reader.stream())