From 5a68d21e8c87e05f2ad0c18e6c7c505f5e9fc93d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 19 Oct 2016 15:08:35 +1300 Subject: [PATCH 1/2] Remove flow module entirely, move contents to top level mitmproxy.flow.io -> mitmproxy.io mitmproxy.flow.export -> mitmproxy.export --- examples/flowbasic | 5 ++--- examples/flowwriter.py | 5 ++--- examples/read_dumpfile | 2 +- mitmproxy/addons/clientplayback.py | 7 +++++-- mitmproxy/addons/filestreamer.py | 2 +- mitmproxy/addons/serverplayback.py | 6 ++++-- mitmproxy/console/common.py | 2 +- mitmproxy/console/flowlist.py | 2 +- mitmproxy/console/flowview.py | 2 +- mitmproxy/console/master.py | 6 +++--- mitmproxy/dump.py | 4 ++-- mitmproxy/{flow => }/export.py | 0 mitmproxy/flow/__init__.py | 7 ------- mitmproxy/{flow => }/io.py | 2 +- mitmproxy/{flow => }/io_compat.py | 0 mitmproxy/master.py | 2 +- mitmproxy/web/app.py | 6 +++--- test/mitmproxy/addons/test_filestreamer.py | 2 +- test/mitmproxy/mastertest.py | 4 ++-- test/mitmproxy/test_dump.py | 7 ++++--- test/mitmproxy/test_flow.py | 19 ++++++++++--------- test/mitmproxy/test_flow_export.py | 2 +- test/mitmproxy/test_flow_format_compat.py | 10 +++++----- 23 files changed, 51 insertions(+), 53 deletions(-) rename mitmproxy/{flow => }/export.py (100%) delete mode 100644 mitmproxy/flow/__init__.py rename mitmproxy/{flow => }/io.py (98%) rename mitmproxy/{flow => }/io_compat.py (100%) diff --git a/examples/flowbasic b/examples/flowbasic index bac98916b..cb1e4ea4f 100755 --- a/examples/flowbasic +++ b/examples/flowbasic @@ -8,7 +8,7 @@ Note that request and response messages are not automatically replied to, so we need to implement handlers to do this. """ -from mitmproxy import flow, controller, options +from mitmproxy import controller, options, master from mitmproxy.proxy import ProxyServer, ProxyConfig @@ -37,7 +37,6 @@ class MyMaster(master.Master): opts = options.Options(cadir="~/.mitmproxy/") config = ProxyConfig(opts) -state = state.State() server = ProxyServer(config) -m = MyMaster(opts, server, state) +m = MyMaster(opts, server) m.run() diff --git a/examples/flowwriter.py b/examples/flowwriter.py index df2e5a40f..afce85aa3 100644 --- a/examples/flowwriter.py +++ b/examples/flowwriter.py @@ -1,7 +1,6 @@ import random import sys - -from mitmproxy.flow import FlowWriter +from mimtproxy import io class Writer: @@ -10,7 +9,7 @@ class Writer: f = sys.stdout else: f = open(path, "wb") - self.w = FlowWriter(f) + self.w = io.FlowWriter(f) def response(self, flow): if random.choice([True, False]): diff --git a/examples/read_dumpfile b/examples/read_dumpfile index b1001c3d4..e0e9064a5 100644 --- a/examples/read_dumpfile +++ b/examples/read_dumpfile @@ -9,7 +9,7 @@ import pprint import sys with open(sys.argv[1], "rb") as logfile: - freader = flow.FlowReader(logfile) + freader = io.FlowReader(logfile) pp = pprint.PrettyPrinter(indent=4) try: for f in freader.stream(): diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 75ef2ffdc..848d07c31 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -1,4 +1,7 @@ -from mitmproxy import exceptions, flow, ctx +from mitmproxy import exceptions +from mitmproxy import ctx +from mitmproxy import io + class ClientPlayback: @@ -21,7 +24,7 @@ class ClientPlayback: if options.client_replay: ctx.log.info(options.client_replay) try: - flows = flow.read_flows_from_paths(options.client_replay) + flows = io.read_flows_from_paths(options.client_replay) except exceptions.FlowReadException as e: raise exceptions.OptionsError(str(e)) self.load(flows) diff --git a/mitmproxy/addons/filestreamer.py b/mitmproxy/addons/filestreamer.py index b1643b212..552d936fd 100644 --- a/mitmproxy/addons/filestreamer.py +++ b/mitmproxy/addons/filestreamer.py @@ -2,7 +2,7 @@ import os.path from mitmproxy import exceptions from mitmproxy import flowfilter -from mitmproxy.flow import io +from mitmproxy import io class FileStreamer: diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 383e27547..1161ce234 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -2,7 +2,9 @@ import urllib import hashlib from netlib import strutils -from mitmproxy import exceptions, flow, ctx +from mitmproxy import exceptions +from mitmproxy import ctx +from mitmproxy import io class ServerPlayback: @@ -91,7 +93,7 @@ class ServerPlayback: self.clear() if options.server_replay: try: - flows = flow.read_flows_from_paths(options.server_replay) + flows = io.read_flows_from_paths(options.server_replay) except exceptions.FlowReadException as e: raise exceptions.OptionsError(str(e)) self.load(flows) diff --git a/mitmproxy/console/common.py b/mitmproxy/console/common.py index 5e335c715..91253668f 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/console/common.py @@ -9,7 +9,7 @@ import urwid.util import netlib from mitmproxy import utils from mitmproxy.console import signals -from mitmproxy.flow import export +from mitmproxy import export from netlib import human try: diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/console/flowlist.py index ba555647c..09a5d027f 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/console/flowlist.py @@ -4,7 +4,7 @@ import netlib.http.url from mitmproxy import exceptions from mitmproxy.console import common from mitmproxy.console import signals -from mitmproxy.flow import export +from mitmproxy import export def _mkhelp(): diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index b8f91bdb5..19afcdbc9 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -15,7 +15,7 @@ from mitmproxy.console import grideditor from mitmproxy.console import searchable from mitmproxy.console import signals from mitmproxy.console import tabs -from mitmproxy.flow import export +from mitmproxy import export from netlib.http import Headers from netlib.http import status_codes diff --git a/mitmproxy/console/master.py b/mitmproxy/console/master.py index 46dd82542..4921373fa 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/console/master.py @@ -19,7 +19,7 @@ from mitmproxy import contentviews from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import master -from mitmproxy import flow +from mitmproxy import io from mitmproxy import flowfilter from mitmproxy import utils from mitmproxy.addons import state @@ -346,7 +346,7 @@ class ConsoleMaster(master.Master): - a list of flows, otherwise. """ try: - return flow.read_flows_from_paths(path) + return io.read_flows_from_paths(path) except exceptions.FlowReadException as e: signals.status_message.send(message=str(e)) @@ -580,7 +580,7 @@ class ConsoleMaster(master.Master): path = os.path.expanduser(path) try: f = open(path, "wb") - fw = flow.FlowWriter(f) + fw = io.FlowWriter(f) for i in flows: fw.add(i) f.close() diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index e8b3126f3..47f693039 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -3,8 +3,8 @@ from typing import Optional from mitmproxy import controller from mitmproxy import exceptions -from mitmproxy import flow from mitmproxy import addons +from mitmproxy import io from mitmproxy import options from mitmproxy import master from mitmproxy.addons import dumper, termlog @@ -68,7 +68,7 @@ class DumpMaster(master.Master): or raises a DumpError if that fails. """ try: - return flow.read_flows_from_paths(paths) + return io.read_flows_from_paths(paths) except exceptions.FlowReadException as e: raise DumpError(str(e)) diff --git a/mitmproxy/flow/export.py b/mitmproxy/export.py similarity index 100% rename from mitmproxy/flow/export.py rename to mitmproxy/export.py diff --git a/mitmproxy/flow/__init__.py b/mitmproxy/flow/__init__.py deleted file mode 100644 index 69e3ed085..000000000 --- a/mitmproxy/flow/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from mitmproxy.flow import export -from mitmproxy.flow.io import FlowWriter, FilteredFlowWriter, FlowReader, read_flows_from_paths - -__all__ = [ - "export", - "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths", -] diff --git a/mitmproxy/flow/io.py b/mitmproxy/io.py similarity index 98% rename from mitmproxy/flow/io.py rename to mitmproxy/io.py index e2a6472c6..c1b7168b6 100644 --- a/mitmproxy/flow/io.py +++ b/mitmproxy/io.py @@ -4,7 +4,7 @@ from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import models from mitmproxy.contrib import tnetstring -from mitmproxy.flow import io_compat +from mitmproxy import io_compat class FlowWriter: diff --git a/mitmproxy/flow/io_compat.py b/mitmproxy/io_compat.py similarity index 100% rename from mitmproxy/flow/io_compat.py rename to mitmproxy/io_compat.py diff --git a/mitmproxy/master.py b/mitmproxy/master.py index aa9892e59..dd98c04c1 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -11,7 +11,7 @@ from mitmproxy import events from mitmproxy import exceptions from mitmproxy import models from mitmproxy import log -from mitmproxy.flow import io +from mitmproxy import io from mitmproxy.protocol import http_replay from netlib import basethread from netlib import http diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index a81d04beb..c45b32a96 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -9,11 +9,11 @@ import hashlib import tornado.websocket import tornado.web from io import BytesIO -from mitmproxy.flow import FlowWriter, FlowReader from mitmproxy import flowfilter from mitmproxy import models from mitmproxy import contentviews +from mitmproxy import io from netlib import version @@ -193,7 +193,7 @@ class DumpFlows(RequestHandler): self.set_header("Content-Type", "application/octet-stream") bio = BytesIO() - fw = FlowWriter(bio) + fw = io.FlowWriter(bio) for f in self.state.flows: fw.add(f) @@ -205,7 +205,7 @@ class DumpFlows(RequestHandler): content = self.request.files.values()[0][0].body bio = BytesIO(content) - self.state.load_flows(FlowReader(bio).stream()) + self.state.load_flows(io.FlowReader(bio).stream()) bio.close() diff --git a/test/mitmproxy/addons/test_filestreamer.py b/test/mitmproxy/addons/test_filestreamer.py index a85fcd0c2..64208f840 100644 --- a/test/mitmproxy/addons/test_filestreamer.py +++ b/test/mitmproxy/addons/test_filestreamer.py @@ -4,7 +4,7 @@ import os.path from mitmproxy.addons import filestreamer from mitmproxy import master -from mitmproxy.flow import io +from mitmproxy import io from mitmproxy import options from mitmproxy import proxy diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 915f9501a..f82eec404 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -4,7 +4,7 @@ from . import tutils import netlib.tutils from mitmproxy import master -from mitmproxy import flow, proxy, models, options +from mitmproxy import io, proxy, models, options class TestMaster: @@ -33,7 +33,7 @@ class MasterTest: def flowfile(self, path): f = open(path, "wb") - fw = flow.FlowWriter(f) + fw = io.FlowWriter(f) t = tutils.tflow(resp=True) fw.add(t) f.close() diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index a81fbd27e..c062a0688 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -1,7 +1,8 @@ import os import io -from mitmproxy import dump, flow, exceptions, proxy +import mitmproxy.io +from mitmproxy import dump, exceptions, proxy from . import tutils, mastertest @@ -127,7 +128,7 @@ class TestDumpMaster(mastertest.MasterTest): self.dummy_cycle( self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, b"" ) - assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 1 + assert len(list(mitmproxy.io.FlowReader(open(p, "rb")).stream())) == 1 def test_write_append(self): with tutils.tmpdir() as d: @@ -140,7 +141,7 @@ class TestDumpMaster(mastertest.MasterTest): self.mkmaster(None, outfile=(p, "ab"), verbosity=0), 1, b"" ) - assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 2 + assert len(list(mitmproxy.io.FlowReader(open(p, "rb")).stream())) == 2 def test_write_err(self): tutils.raises( diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 86cd7d160..d33d395b5 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -3,7 +3,8 @@ import io import netlib.utils from netlib.http import Headers -from mitmproxy import flowfilter, flow, options +import mitmproxy.io +from mitmproxy import flowfilter, options from mitmproxy.addons import state from mitmproxy.contrib import tnetstring from mitmproxy.exceptions import FlowReadException, Kill @@ -329,7 +330,7 @@ class TestSerialize: def _treader(self): sio = io.BytesIO() - w = flow.FlowWriter(sio) + w = mitmproxy.io.FlowWriter(sio) for i in range(3): f = tutils.tflow(resp=True) w.add(f) @@ -342,18 +343,18 @@ class TestSerialize: w.add(f) sio.seek(0) - return flow.FlowReader(sio) + return mitmproxy.io.FlowReader(sio) def test_roundtrip(self): sio = io.BytesIO() f = tutils.tflow() f.marked = True f.request.content = bytes(bytearray(range(256))) - w = flow.FlowWriter(sio) + w = mitmproxy.io.FlowWriter(sio) w.add(f) sio.seek(0) - r = flow.FlowReader(sio) + r = mitmproxy.io.FlowReader(sio) l = list(r.stream()) assert len(l) == 1 @@ -386,7 +387,7 @@ class TestSerialize: def test_filter(self): sio = io.BytesIO() flt = flowfilter.parse("~c 200") - w = flow.FilteredFlowWriter(sio, flt) + w = mitmproxy.io.FilteredFlowWriter(sio, flt) f = tutils.tflow(resp=True) f.response.status_code = 200 @@ -397,14 +398,14 @@ class TestSerialize: w.add(f) sio.seek(0) - r = flow.FlowReader(sio) + r = mitmproxy.io.FlowReader(sio) assert len(list(r.stream())) def test_error(self): sio = io.BytesIO() sio.write(b"bogus") sio.seek(0) - r = flow.FlowReader(sio) + r = mitmproxy.io.FlowReader(sio) tutils.raises(FlowReadException, list, r.stream()) f = FlowReadException("foo") @@ -418,7 +419,7 @@ class TestSerialize: tnetstring.dump(d, sio) sio.seek(0) - r = flow.FlowReader(sio) + r = mitmproxy.io.FlowReader(sio) tutils.raises("version", list, r.stream()) diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 920f538b7..aafd5a1c0 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -2,7 +2,7 @@ import re import netlib.tutils from netlib.http import Headers -from mitmproxy.flow import export # heh +from mitmproxy import export # heh from . import tutils diff --git a/test/mitmproxy/test_flow_format_compat.py b/test/mitmproxy/test_flow_format_compat.py index cc80db81a..4997aff9d 100644 --- a/test/mitmproxy/test_flow_format_compat.py +++ b/test/mitmproxy/test_flow_format_compat.py @@ -1,11 +1,11 @@ -from mitmproxy.flow import FlowReader -from mitmproxy.exceptions import FlowReadException +from mitmproxy import io +from mitmproxy import exceptions from . import tutils def test_load(): with open(tutils.test_data.path("data/dumpfile-011"), "rb") as f: - flow_reader = FlowReader(f) + flow_reader = io.FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 assert flows[0].request.url == "https://example.com/" @@ -13,6 +13,6 @@ def test_load(): def test_cannot_convert(): with open(tutils.test_data.path("data/dumpfile-010"), "rb") as f: - flow_reader = FlowReader(f) - with tutils.raises(FlowReadException): + flow_reader = io.FlowReader(f) + with tutils.raises(exceptions.FlowReadException): list(flow_reader.stream()) From 24cf8da27eb56a65bf3e4ceb78bbeacdb1864597 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 19 Oct 2016 15:25:39 +1300 Subject: [PATCH 2/2] Move all tools into mitmproxy.tools, move models/* to top level The primary motivation here (and for all the other moving around) is to present a clean "front of house" to library users, and to migrate primary objects to the top of the module hierarchy. --- docs/scripting/api.rst | 16 +- examples/flowwriter.py | 2 +- examples/redirect_requests.py | 4 +- mitmproxy/addons/clientplayback.py | 1 - mitmproxy/addons/filestreamer.py | 2 +- mitmproxy/addons/state.py | 4 +- mitmproxy/{models => }/connections.py | 0 mitmproxy/console/__init__.py | 4 - mitmproxy/events.py | 7 +- mitmproxy/{models => }/flow.py | 11 +- mitmproxy/flowfilter.py | 56 +++---- mitmproxy/{models => }/http.py | 2 +- mitmproxy/io.py | 13 +- mitmproxy/master.py | 17 +- mitmproxy/models/__init__.py | 23 --- mitmproxy/protocol/base.py | 12 +- mitmproxy/protocol/http.py | 151 +++++++++--------- mitmproxy/protocol/http1.py | 12 +- mitmproxy/protocol/http2.py | 12 +- mitmproxy/protocol/http_replay.py | 70 ++++---- mitmproxy/protocol/rawtcp.py | 18 +-- mitmproxy/protocol/tls.py | 4 +- mitmproxy/proxy/root_context.py | 2 +- mitmproxy/proxy/server.py | 7 +- mitmproxy/{models => }/tcp.py | 6 +- mitmproxy/tools/__init__.py | 0 mitmproxy/{ => tools}/cmdline.py | 2 +- mitmproxy/tools/console/__init__.py | 4 + mitmproxy/{ => tools}/console/common.py | 2 +- .../{ => tools}/console/flowdetailview.py | 2 +- mitmproxy/{ => tools}/console/flowlist.py | 4 +- mitmproxy/{ => tools}/console/flowview.py | 22 +-- .../console/grideditor/__init__.py | 0 .../{ => tools}/console/grideditor/base.py | 4 +- .../console/grideditor/col_bytes.py | 4 +- .../console/grideditor/col_subgrid.py | 4 +- .../console/grideditor/col_text.py | 4 +- .../{ => tools}/console/grideditor/editors.py | 12 +- mitmproxy/{ => tools}/console/help.py | 4 +- mitmproxy/{ => tools}/console/master.py | 20 +-- mitmproxy/{ => tools}/console/options.py | 10 +- .../{ => tools}/console/palettepicker.py | 8 +- mitmproxy/{ => tools}/console/palettes.py | 0 mitmproxy/{ => tools}/console/pathedit.py | 0 mitmproxy/{ => tools}/console/searchable.py | 2 +- mitmproxy/{ => tools}/console/select.py | 2 +- mitmproxy/{ => tools}/console/signals.py | 0 mitmproxy/{ => tools}/console/statusbar.py | 6 +- mitmproxy/{ => tools}/console/tabs.py | 0 mitmproxy/{ => tools}/console/window.py | 2 +- mitmproxy/{ => tools}/dump.py | 0 mitmproxy/{ => tools}/main.py | 8 +- mitmproxy/tools/web/__init__.py | 2 + mitmproxy/{ => tools}/web/app.py | 9 +- mitmproxy/{ => tools}/web/master.py | 2 +- mitmproxy/{ => tools}/web/static/app.css | 0 mitmproxy/{ => tools}/web/static/app.js | 0 .../web/static/fonts/fontawesome-webfont.eot | Bin .../web/static/fonts/fontawesome-webfont.svg | 0 .../web/static/fonts/fontawesome-webfont.ttf | Bin .../web/static/fonts/fontawesome-webfont.woff | Bin .../web/static/images/chrome-devtools/LICENSE | 0 .../chrome-devtools/resourceCSSIcon.png | Bin .../chrome-devtools/resourceDocumentIcon.png | Bin .../images/chrome-devtools/resourceJSIcon.png | Bin .../chrome-devtools/resourcePlainIcon.png | Bin .../{ => tools}/web/static/images/favicon.ico | Bin .../static/images/resourceExecutableIcon.png | Bin .../web/static/images/resourceFlashIcon.png | Bin .../web/static/images/resourceImageIcon.png | Bin .../web/static/images/resourceJavaIcon.png | Bin .../static/images/resourceNotModifiedIcon.png | Bin .../static/images/resourceRedirectIcon.png | Bin mitmproxy/{ => tools}/web/static/vendor.css | 0 mitmproxy/{ => tools}/web/static/vendor.js | 0 .../{ => tools}/web/templates/index.html | 0 mitmproxy/web/__init__.py | 2 - setup.py | 6 +- test/mitmproxy/addons/test_dumper.py | 6 +- test/mitmproxy/addons/test_termlog.py | 2 +- test/mitmproxy/console/test_common.py | 2 +- test/mitmproxy/console/test_help.py | 2 +- test/mitmproxy/console/test_master.py | 4 +- test/mitmproxy/console/test_palettes.py | 2 +- test/mitmproxy/console/test_pathedit.py | 2 +- test/mitmproxy/mastertest.py | 7 +- test/mitmproxy/test_cmdline.py | 2 +- test/mitmproxy/test_dump.py | 4 +- test/mitmproxy/test_flow.py | 59 ++++--- test/mitmproxy/test_proxy.py | 8 +- test/mitmproxy/test_server.py | 19 +-- test/mitmproxy/test_utils.py | 2 +- test/mitmproxy/test_web_app.py | 4 +- test/mitmproxy/test_web_master.py | 2 +- test/mitmproxy/tutils.py | 29 ++-- 95 files changed, 381 insertions(+), 378 deletions(-) rename mitmproxy/{models => }/connections.py (100%) delete mode 100644 mitmproxy/console/__init__.py rename mitmproxy/{models => }/flow.py (94%) rename mitmproxy/{models => }/http.py (99%) delete mode 100644 mitmproxy/models/__init__.py rename mitmproxy/{models => }/tcp.py (91%) create mode 100644 mitmproxy/tools/__init__.py rename mitmproxy/{ => tools}/cmdline.py (99%) create mode 100644 mitmproxy/tools/console/__init__.py rename mitmproxy/{ => tools}/console/common.py (99%) rename mitmproxy/{ => tools}/console/flowdetailview.py (98%) rename mitmproxy/{ => tools}/console/flowlist.py (99%) rename mitmproxy/{ => tools}/console/flowview.py (97%) rename mitmproxy/{ => tools}/console/grideditor/__init__.py (100%) rename mitmproxy/{ => tools}/console/grideditor/base.py (99%) rename mitmproxy/{ => tools}/console/grideditor/col_bytes.py (96%) rename mitmproxy/{ => tools}/console/grideditor/col_subgrid.py (92%) rename mitmproxy/{ => tools}/console/grideditor/col_text.py (92%) rename mitmproxy/{ => tools}/console/grideditor/editors.py (94%) rename mitmproxy/{ => tools}/console/help.py (97%) rename mitmproxy/{ => tools}/console/master.py (97%) rename mitmproxy/{ => tools}/console/options.py (97%) rename mitmproxy/{ => tools}/console/palettepicker.py (92%) rename mitmproxy/{ => tools}/console/palettes.py (100%) rename mitmproxy/{ => tools}/console/pathedit.py (100%) rename mitmproxy/{ => tools}/console/searchable.py (98%) rename mitmproxy/{ => tools}/console/select.py (98%) rename mitmproxy/{ => tools}/console/signals.py (100%) rename mitmproxy/{ => tools}/console/statusbar.py (98%) rename mitmproxy/{ => tools}/console/tabs.py (100%) rename mitmproxy/{ => tools}/console/window.py (98%) rename mitmproxy/{ => tools}/dump.py (100%) rename mitmproxy/{ => tools}/main.py (96%) create mode 100644 mitmproxy/tools/web/__init__.py rename mitmproxy/{ => tools}/web/app.py (98%) rename mitmproxy/{ => tools}/web/master.py (99%) rename mitmproxy/{ => tools}/web/static/app.css (100%) rename mitmproxy/{ => tools}/web/static/app.js (100%) rename mitmproxy/{ => tools}/web/static/fonts/fontawesome-webfont.eot (100%) rename mitmproxy/{ => tools}/web/static/fonts/fontawesome-webfont.svg (100%) rename mitmproxy/{ => tools}/web/static/fonts/fontawesome-webfont.ttf (100%) rename mitmproxy/{ => tools}/web/static/fonts/fontawesome-webfont.woff (100%) rename mitmproxy/{ => tools}/web/static/images/chrome-devtools/LICENSE (100%) rename mitmproxy/{ => tools}/web/static/images/chrome-devtools/resourceCSSIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/chrome-devtools/resourceDocumentIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/chrome-devtools/resourceJSIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/chrome-devtools/resourcePlainIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/favicon.ico (100%) rename mitmproxy/{ => tools}/web/static/images/resourceExecutableIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/resourceFlashIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/resourceImageIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/resourceJavaIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/resourceNotModifiedIcon.png (100%) rename mitmproxy/{ => tools}/web/static/images/resourceRedirectIcon.png (100%) rename mitmproxy/{ => tools}/web/static/vendor.css (100%) rename mitmproxy/{ => tools}/web/static/vendor.js (100%) rename mitmproxy/{ => tools}/web/templates/index.html (100%) delete mode 100644 mitmproxy/web/__init__.py diff --git a/docs/scripting/api.rst b/docs/scripting/api.rst index 59ef8d95d..e82afef4f 100644 --- a/docs/scripting/api.rst +++ b/docs/scripting/api.rst @@ -5,11 +5,11 @@ API === - Errors - - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_ + - `mitmproxy.flow.Error <#mitmproxy.flow.Error>`_ - HTTP - - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_ - - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_ - - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_ + - `mitmproxy.http.HTTPRequest <#mitmproxy.http.HTTPRequest>`_ + - `mitmproxy.http.HTTPResponse <#mitmproxy.http.HTTPResponse>`_ + - `mitmproxy.http.HTTPFlow <#mitmproxy.http.HTTPFlow>`_ - Logging - `mitmproxy.log.Log <#mitmproxy.controller.Log>`_ - `mitmproxy.log.LogEntry <#mitmproxy.controller.LogEntry>`_ @@ -18,19 +18,19 @@ API Errors ------ -.. autoclass:: mitmproxy.models.flow.Error +.. autoclass:: mitmproxy.flow.Error :inherited-members: HTTP ---- -.. autoclass:: mitmproxy.models.http.HTTPRequest +.. autoclass:: mitmproxy.http.HTTPRequest :inherited-members: -.. autoclass:: mitmproxy.models.http.HTTPResponse +.. autoclass:: mitmproxy.http.HTTPResponse :inherited-members: -.. autoclass:: mitmproxy.models.http.HTTPFlow +.. autoclass:: mitmproxy.http.HTTPFlow :inherited-members: Logging diff --git a/examples/flowwriter.py b/examples/flowwriter.py index afce85aa3..a9768542e 100644 --- a/examples/flowwriter.py +++ b/examples/flowwriter.py @@ -1,6 +1,6 @@ import random import sys -from mimtproxy import io +from mitmproxy import io class Writer: diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py index bbb84e2f1..c28042db1 100644 --- a/examples/redirect_requests.py +++ b/examples/redirect_requests.py @@ -1,7 +1,7 @@ """ This example shows two ways to redirect flows to other destinations. """ -from mitmproxy.models import HTTPResponse +from mitmproxy import http def request(flow): @@ -11,7 +11,7 @@ def request(flow): # Method 1: Answer with a locally generated response if flow.request.pretty_host.endswith("example.com"): - flow.response = HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"}) + flow.response = http.HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"}) # Method 2: Redirect the request to a different server if flow.request.pretty_host.endswith("example.org"): diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 848d07c31..7f2b53ac6 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -3,7 +3,6 @@ from mitmproxy import ctx from mitmproxy import io - class ClientPlayback: def __init__(self): self.flows = None diff --git a/mitmproxy/addons/filestreamer.py b/mitmproxy/addons/filestreamer.py index 552d936fd..031b44ab6 100644 --- a/mitmproxy/addons/filestreamer.py +++ b/mitmproxy/addons/filestreamer.py @@ -8,7 +8,7 @@ from mitmproxy import io class FileStreamer: def __init__(self): self.stream = None - self.active_flows = set() # type: Set[models.Flow] + self.active_flows = set() # type: Set[flow.Flow] def start_stream_to_path(self, path, mode, flt): path = os.path.expanduser(path) diff --git a/mitmproxy/addons/state.py b/mitmproxy/addons/state.py index bb7460b6f..ae7338ccd 100644 --- a/mitmproxy/addons/state.py +++ b/mitmproxy/addons/state.py @@ -3,12 +3,12 @@ from abc import abstractmethod, ABCMeta from typing import List # noqa from mitmproxy import flowfilter -from mitmproxy import models # noqa +from mitmproxy import flow # noqa class FlowList(metaclass=ABCMeta): def __init__(self): - self._list = [] # type: List[models.Flow] + self._list = [] # type: List[flow.Flow] def __iter__(self): return iter(self._list) diff --git a/mitmproxy/models/connections.py b/mitmproxy/connections.py similarity index 100% rename from mitmproxy/models/connections.py rename to mitmproxy/connections.py diff --git a/mitmproxy/console/__init__.py b/mitmproxy/console/__init__.py deleted file mode 100644 index ae23c694c..000000000 --- a/mitmproxy/console/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from mitmproxy.console import master - - -__all__ = ["master"] diff --git a/mitmproxy/events.py b/mitmproxy/events.py index 0e512aaec..56f1a45b8 100644 --- a/mitmproxy/events.py +++ b/mitmproxy/events.py @@ -1,5 +1,6 @@ from mitmproxy import controller -from mitmproxy import models +from mitmproxy import http +from mitmproxy import tcp Events = frozenset([ "clientconnect", @@ -34,7 +35,7 @@ Events = frozenset([ def event_sequence(f): - if isinstance(f, models.HTTPFlow): + if isinstance(f, http.HTTPFlow): if f.request: yield "requestheaders", f yield "request", f @@ -43,7 +44,7 @@ def event_sequence(f): yield "response", f if f.error: yield "error", f - elif isinstance(f, models.TCPFlow): + elif isinstance(f, tcp.TCPFlow): messages = f.messages f.messages = [] f.reply = controller.DummyReply() diff --git a/mitmproxy/models/flow.py b/mitmproxy/flow.py similarity index 94% rename from mitmproxy/models/flow.py rename to mitmproxy/flow.py index 2596165b2..13b852efb 100644 --- a/mitmproxy/models/flow.py +++ b/mitmproxy/flow.py @@ -3,8 +3,7 @@ import copy import uuid from mitmproxy import stateobject -from mitmproxy.models.connections import ClientConnection -from mitmproxy.models.connections import ServerConnection +from mitmproxy import connections from netlib import version from typing import Optional # noqa @@ -68,8 +67,8 @@ class Flow(stateobject.StateObject): def __init__( self, type: str, - client_conn: ClientConnection, - server_conn: ServerConnection, + client_conn: connections.ClientConnection, + server_conn: connections.ServerConnection, live=None ): self.type = type @@ -87,8 +86,8 @@ class Flow(stateobject.StateObject): _stateobject_attributes = dict( id=str, error=Error, - client_conn=ClientConnection, - server_conn=ServerConnection, + client_conn=connections.ClientConnection, + server_conn=connections.ServerConnection, type=str, intercepted=bool, marked=bool, diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index b63c27d92..f1454fd19 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -36,9 +36,9 @@ import re import sys import functools -from mitmproxy.models.http import HTTPFlow -from mitmproxy.models.tcp import TCPFlow -from mitmproxy.models.flow import Flow +from mitmproxy import http +from mitmproxy import tcp +from mitmproxy import flow from netlib import strutils @@ -94,7 +94,7 @@ class FHTTP(_Action): code = "http" help = "Match HTTP flows" - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): return True @@ -103,7 +103,7 @@ class FTCP(_Action): code = "tcp" help = "Match TCP flows" - @only(TCPFlow) + @only(tcp.TCPFlow) def __call__(self, f): return True @@ -112,7 +112,7 @@ class FReq(_Action): code = "q" help = "Match request with no response" - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if not f.response: return True @@ -122,7 +122,7 @@ class FResp(_Action): code = "s" help = "Match response" - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): return bool(f.response) @@ -162,7 +162,7 @@ class FAsset(_Action): ] ASSET_TYPES = [re.compile(x) for x in ASSET_TYPES] - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if f.response: for i in self.ASSET_TYPES: @@ -175,7 +175,7 @@ class FContentType(_Rex): code = "t" help = "Content-type header" - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if _check_content_type(self.re, f.request): return True @@ -188,7 +188,7 @@ class FContentTypeRequest(_Rex): code = "tq" help = "Request Content-Type header" - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): return _check_content_type(self.re, f.request) @@ -197,7 +197,7 @@ class FContentTypeResponse(_Rex): code = "ts" help = "Response Content-Type header" - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if f.response: return _check_content_type(self.re, f.response) @@ -209,7 +209,7 @@ class FHead(_Rex): help = "Header" flags = re.MULTILINE - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if f.request and self.re.search(bytes(f.request.headers)): return True @@ -223,7 +223,7 @@ class FHeadRequest(_Rex): help = "Request header" flags = re.MULTILINE - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if f.request and self.re.search(bytes(f.request.headers)): return True @@ -234,7 +234,7 @@ class FHeadResponse(_Rex): help = "Response header" flags = re.MULTILINE - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if f.response and self.re.search(bytes(f.response.headers)): return True @@ -245,16 +245,16 @@ class FBod(_Rex): help = "Body" flags = re.DOTALL - @only(HTTPFlow, TCPFlow) + @only(http.HTTPFlow, tcp.TCPFlow) def __call__(self, f): - if isinstance(f, HTTPFlow): + if isinstance(f, http.HTTPFlow): if f.request and f.request.raw_content: if self.re.search(f.request.get_content(strict=False)): return True if f.response and f.response.raw_content: if self.re.search(f.response.get_content(strict=False)): return True - elif isinstance(f, TCPFlow): + elif isinstance(f, tcp.TCPFlow): for msg in f.messages: if self.re.search(msg.content): return True @@ -266,13 +266,13 @@ class FBodRequest(_Rex): help = "Request body" flags = re.DOTALL - @only(HTTPFlow, TCPFlow) + @only(http.HTTPFlow, tcp.TCPFlow) def __call__(self, f): - if isinstance(f, HTTPFlow): + if isinstance(f, http.HTTPFlow): if f.request and f.request.raw_content: if self.re.search(f.request.get_content(strict=False)): return True - elif isinstance(f, TCPFlow): + elif isinstance(f, tcp.TCPFlow): for msg in f.messages: if msg.from_client and self.re.search(msg.content): return True @@ -283,13 +283,13 @@ class FBodResponse(_Rex): help = "Response body" flags = re.DOTALL - @only(HTTPFlow, TCPFlow) + @only(http.HTTPFlow, tcp.TCPFlow) def __call__(self, f): - if isinstance(f, HTTPFlow): + if isinstance(f, http.HTTPFlow): if f.response and f.response.raw_content: if self.re.search(f.response.get_content(strict=False)): return True - elif isinstance(f, TCPFlow): + elif isinstance(f, tcp.TCPFlow): for msg in f.messages: if not msg.from_client and self.re.search(msg.content): return True @@ -300,7 +300,7 @@ class FMethod(_Rex): help = "Method" flags = re.IGNORECASE - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): return bool(self.re.search(f.request.data.method)) @@ -310,7 +310,7 @@ class FDomain(_Rex): help = "Domain" flags = re.IGNORECASE - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): return bool(self.re.search(f.request.data.host)) @@ -327,7 +327,7 @@ class FUrl(_Rex): toks = toks[1:] return klass(*toks) - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): return self.re.search(f.request.url) @@ -360,7 +360,7 @@ class FCode(_Int): code = "c" help = "HTTP response code" - @only(HTTPFlow) + @only(http.HTTPFlow) def __call__(self, f): if f.response and f.response.status_code == self.num: return True @@ -485,7 +485,7 @@ def _make(): bnf = _make() -TFilter = Callable[[Flow], bool] +TFilter = Callable[[flow.Flow], bool] def parse(s: str) -> TFilter: diff --git a/mitmproxy/models/http.py b/mitmproxy/http.py similarity index 99% rename from mitmproxy/models/http.py rename to mitmproxy/http.py index 91263b95f..7fe70f9bd 100644 --- a/mitmproxy/models/http.py +++ b/mitmproxy/http.py @@ -1,6 +1,6 @@ import cgi -from mitmproxy.models import flow +from mitmproxy import flow from netlib import http from netlib import version from netlib import tcp diff --git a/mitmproxy/io.py b/mitmproxy/io.py index c1b7168b6..27ffa036d 100644 --- a/mitmproxy/io.py +++ b/mitmproxy/io.py @@ -2,11 +2,18 @@ import os from mitmproxy import exceptions from mitmproxy import flowfilter -from mitmproxy import models +from mitmproxy import http +from mitmproxy import tcp from mitmproxy.contrib import tnetstring from mitmproxy import io_compat +FLOW_TYPES = dict( + http=http.HTTPFlow, + tcp=tcp.TCPFlow, +) + + class FlowWriter: def __init__(self, fo): self.fo = fo @@ -31,9 +38,9 @@ class FlowReader: data = io_compat.migrate_flow(data) except ValueError as e: raise exceptions.FlowReadException(str(e)) - if data["type"] not in models.FLOW_TYPES: + if data["type"] not in FLOW_TYPES: raise exceptions.FlowReadException("Unknown flow type: {}".format(data["type"])) - yield models.FLOW_TYPES[data["type"]].from_state(data) + yield FLOW_TYPES[data["type"]].from_state(data) except ValueError as e: if str(e) == "not a tnetstring: empty file": return # Error is due to EOF diff --git a/mitmproxy/master.py b/mitmproxy/master.py index dd98c04c1..672ff1e8c 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -9,12 +9,13 @@ from mitmproxy import options from mitmproxy import controller from mitmproxy import events from mitmproxy import exceptions -from mitmproxy import models +from mitmproxy import connections +from mitmproxy import http from mitmproxy import log from mitmproxy import io from mitmproxy.protocol import http_replay from netlib import basethread -from netlib import http +import netlib.http from . import ctx as mitmproxy_ctx @@ -117,13 +118,13 @@ class Master: """ this method creates a new artificial and minimalist request also adds it to flowlist """ - c = models.ClientConnection.make_dummy(("", 0)) - s = models.ServerConnection.make_dummy((host, port)) + c = connections.ClientConnection.make_dummy(("", 0)) + s = connections.ServerConnection.make_dummy((host, port)) - f = models.HTTPFlow(c, s) - headers = http.Headers() + f = http.HTTPFlow(c, s) + headers = netlib.http.Headers() - req = models.HTTPRequest( + req = http.HTTPRequest( "absolute", method, scheme, @@ -142,7 +143,7 @@ class Master: """ Loads a flow """ - if isinstance(f, models.HTTPFlow): + if isinstance(f, http.HTTPFlow): if self.server and self.options.mode == "reverse": f.request.host = self.server.config.upstream_server.address.host f.request.port = self.server.config.upstream_server.address.port diff --git a/mitmproxy/models/__init__.py b/mitmproxy/models/__init__.py deleted file mode 100644 index 0dcc73a0d..000000000 --- a/mitmproxy/models/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -from netlib.http import decoded -from .connections import ClientConnection, ServerConnection -from .flow import Flow, Error -from .http import ( - HTTPFlow, HTTPRequest, HTTPResponse, - make_error_response, make_connect_request, make_connect_response, expect_continue_response -) -from .tcp import TCPFlow - -FLOW_TYPES = dict( - http=HTTPFlow, - tcp=TCPFlow, -) - -__all__ = [ - "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded", - "make_error_response", "make_connect_request", - "make_connect_response", "expect_continue_response", - "ClientConnection", "ServerConnection", - "Flow", "Error", - "TCPFlow", - "FLOW_TYPES", -] diff --git a/mitmproxy/protocol/base.py b/mitmproxy/protocol/base.py index d280b69ca..53cfd1377 100644 --- a/mitmproxy/protocol/base.py +++ b/mitmproxy/protocol/base.py @@ -1,6 +1,6 @@ import netlib.exceptions from mitmproxy import exceptions -from mitmproxy import models +from mitmproxy import connections class _LayerCodeCompletion: @@ -16,9 +16,9 @@ class _LayerCodeCompletion: self.config = None """@type: mitmproxy.proxy.ProxyConfig""" self.client_conn = None - """@type: mitmproxy.models.ClientConnection""" + """@type: mitmproxy.connections.ClientConnection""" self.server_conn = None - """@type: mitmproxy.models.ServerConnection""" + """@type: mitmproxy.connections.ServerConnection""" self.channel = None """@type: mitmproxy.controller.Channel""" self.ctx = None @@ -111,10 +111,10 @@ class ServerConnectionMixin: self.server_conn = None if self.config.options.spoof_source_address: - self.server_conn = models.ServerConnection( + self.server_conn = connections.ServerConnection( server_address, (self.ctx.client_conn.address.host, 0), True) else: - self.server_conn = models.ServerConnection( + self.server_conn = connections.ServerConnection( server_address, (self.config.options.listen_host, 0)) self.__check_self_connect() @@ -157,7 +157,7 @@ class ServerConnectionMixin: self.server_conn.close() self.channel.tell("serverdisconnect", self.server_conn) - self.server_conn = models.ServerConnection( + self.server_conn = connections.ServerConnection( address, (self.server_conn.source_address.host, 0), self.config.options.spoof_source_address diff --git a/mitmproxy/protocol/http.py b/mitmproxy/protocol/http.py index b9abfb4c3..325bf815f 100644 --- a/mitmproxy/protocol/http.py +++ b/mitmproxy/protocol/http.py @@ -3,10 +3,11 @@ import netlib.exceptions import time import traceback from mitmproxy import exceptions -from mitmproxy import models +from mitmproxy import http +from mitmproxy import flow from mitmproxy.protocol import base from mitmproxy.protocol import websockets as pwebsockets -from netlib import http +import netlib.http from netlib import tcp from netlib import websockets @@ -55,7 +56,7 @@ class _HttpTransmissionLayer(base.Layer): def send_response_body(self, response, chunks): raise NotImplementedError() - def check_close_connection(self, flow): + def check_close_connection(self, f): raise NotImplementedError() @@ -140,9 +141,9 @@ class HttpLayer(base.Layer): self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: - flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self) + f = http.HTTPFlow(self.client_conn, self.server_conn, live=self) try: - request = self.get_request_from_client(flow) + request = self.get_request_from_client(f) # Make sure that the incoming request matches our expectations self.validate_request(request) except netlib.exceptions.HttpReadDisconnect: @@ -165,7 +166,7 @@ class HttpLayer(base.Layer): if not (self.http_authenticated or self.authenticate(request)): return - flow.request = request + f.request = request try: # Regular Proxy Mode: Handle CONNECT @@ -176,74 +177,74 @@ class HttpLayer(base.Layer): # HTTPS tasting means that ordinary errors like resolution and # connection errors can happen here. self.send_error_response(502, repr(e)) - flow.error = models.Error(str(e)) - self.channel.ask("error", flow) + f.error = flow.Error(str(e)) + self.channel.ask("error", f) return # update host header in reverse proxy mode if self.config.options.mode == "reverse": - flow.request.headers["Host"] = self.config.upstream_server.address.host + f.request.headers["Host"] = self.config.upstream_server.address.host # set upstream auth if self.mode == "upstream" and self.config.upstream_auth is not None: - flow.request.headers["Proxy-Authorization"] = self.config.upstream_auth - self.process_request_hook(flow) + f.request.headers["Proxy-Authorization"] = self.config.upstream_auth + self.process_request_hook(f) try: if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): # We only support RFC6455 with WebSockets version 13 # allow inline scripts to manipulate the client handshake - self.channel.ask("websocket_handshake", flow) + self.channel.ask("websocket_handshake", f) - if not flow.response: + if not f.response: self.establish_server_connection( - flow.request.host, - flow.request.port, - flow.request.scheme + f.request.host, + f.request.port, + f.request.scheme ) - self.get_response_from_server(flow) + self.get_response_from_server(f) else: # response was set by an inline script. # we now need to emulate the responseheaders hook. - self.channel.ask("responseheaders", flow) + self.channel.ask("responseheaders", f) - self.log("response", "debug", [repr(flow.response)]) - self.channel.ask("response", flow) - self.send_response_to_client(flow) + self.log("response", "debug", [repr(f.response)]) + self.channel.ask("response", f) + self.send_response_to_client(f) - if self.check_close_connection(flow): + if self.check_close_connection(f): return # Handle 101 Switching Protocols - if flow.response.status_code == 101: - return self.handle_101_switching_protocols(flow) + if f.response.status_code == 101: + return self.handle_101_switching_protocols(f) # Upstream Proxy Mode: Handle CONNECT - if flow.request.first_line_format == "authority" and flow.response.status_code == 200: - self.handle_upstream_mode_connect(flow.request.copy()) + if f.request.first_line_format == "authority" and f.response.status_code == 200: + self.handle_upstream_mode_connect(f.request.copy()) return except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) - if not flow.response: - flow.error = models.Error(str(e)) - self.channel.ask("error", flow) + if not f.response: + f.error = flow.Error(str(e)) + self.channel.ask("error", f) return else: raise exceptions.ProtocolException( "Error in HTTP connection: %s" % repr(e) ) finally: - if flow: - flow.live = False + if f: + f.live = False - def get_request_from_client(self, flow): + def get_request_from_client(self, f): request = self.read_request() - flow.request = request - self.channel.ask("requestheaders", flow) + f.request = request + self.channel.ask("requestheaders", f) if request.headers.get("expect", "").lower() == "100-continue": # TODO: We may have to use send_response_headers for HTTP2 here. - self.send_response(models.expect_continue_response) + self.send_response(http.expect_continue_response) request.headers.pop("expect") request.body = b"".join(self.read_request_body(request)) request.timestamp_end = time.time() @@ -251,7 +252,7 @@ class HttpLayer(base.Layer): def send_error_response(self, code, message, headers=None): try: - response = models.make_error_response(code, message, headers) + response = http.make_error_response(code, message, headers) self.send_response(response) except (netlib.exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException): self.log(traceback.format_exc(), "debug") @@ -265,7 +266,7 @@ class HttpLayer(base.Layer): def handle_regular_mode_connect(self, request): self.http_authenticated = True self.set_server((request.host, request.port)) - self.send_response(models.make_connect_response(request.data.http_version)) + self.send_response(http.make_connect_response(request.data.http_version)) layer = self.ctx.next_layer(self) layer() @@ -273,29 +274,29 @@ class HttpLayer(base.Layer): layer = UpstreamConnectLayer(self, connect_request) layer() - def send_response_to_client(self, flow): - if not flow.response.stream: + def send_response_to_client(self, f): + if not f.response.stream: # no streaming: # we already received the full response from the server and can # send it to the client straight away. - self.send_response(flow.response) + self.send_response(f.response) else: # streaming: # First send the headers and then transfer the response incrementally - self.send_response_headers(flow.response) + self.send_response_headers(f.response) chunks = self.read_response_body( - flow.request, - flow.response + f.request, + f.response ) - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - self.send_response_body(flow.response, chunks) - flow.response.timestamp_end = time.time() + if callable(f.response.stream): + chunks = f.response.stream(chunks) + self.send_response_body(f.response, chunks) + f.response.timestamp_end = time.time() - def get_response_from_server(self, flow): + def get_response_from_server(self, f): def get_response(): - self.send_request(flow.request) - flow.response = self.read_response_headers() + self.send_request(f.request) + f.response = self.read_response_headers() try: get_response() @@ -325,23 +326,23 @@ class HttpLayer(base.Layer): get_response() # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - self.channel.ask("responseheaders", flow) + # inline script to set f.stream = True + self.channel.ask("responseheaders", f) - if flow.response.stream: - flow.response.data.content = None + if f.response.stream: + f.response.data.content = None else: - flow.response.data.content = b"".join(self.read_response_body( - flow.request, - flow.response + f.response.data.content = b"".join(self.read_response_body( + f.request, + f.response )) - flow.response.timestamp_end = time.time() + f.response.timestamp_end = time.time() # no further manipulation of self.server_conn beyond this point # we can safely set it as the final attribute value here. - flow.server_conn = self.server_conn + f.server_conn = self.server_conn - def process_request_hook(self, flow): + def process_request_hook(self, f): # Determine .scheme, .host and .port attributes for inline scripts. # For absolute-form requests, they are directly given in the request. # For authority-form requests, we only need to determine the request scheme. @@ -353,13 +354,13 @@ class HttpLayer(base.Layer): pass else: # Setting request.host also updates the host header, which we want to preserve - host_header = flow.request.headers.get("host", None) - flow.request.host = self.__initial_server_conn.address.host - flow.request.port = self.__initial_server_conn.address.port + host_header = f.request.headers.get("host", None) + f.request.host = self.__initial_server_conn.address.host + f.request.port = self.__initial_server_conn.address.port if host_header: - flow.request.headers["host"] = host_header - flow.request.scheme = "https" if self.__initial_server_tls else "http" - self.channel.ask("request", flow) + f.request.headers["host"] = host_header + f.request.scheme = "https" if self.__initial_server_tls else "http" + self.channel.ask("request", f) def establish_server_connection(self, host, port, scheme): address = tcp.Address((host, port)) @@ -419,29 +420,29 @@ class HttpLayer(base.Layer): self.config.authenticator.clean(request.headers) else: if self.mode == "transparent": - self.send_response(models.make_error_response( + self.send_response(http.make_error_response( 401, "Authentication Required", - http.Headers(**self.config.authenticator.auth_challenge_headers()) + netlib.http.Headers(**self.config.authenticator.auth_challenge_headers()) )) else: - self.send_response(models.make_error_response( + self.send_response(http.make_error_response( 407, "Proxy Authentication Required", - http.Headers(**self.config.authenticator.auth_challenge_headers()) + netlib.http.Headers(**self.config.authenticator.auth_challenge_headers()) )) return False return True - def handle_101_switching_protocols(self, flow): + def handle_101_switching_protocols(self, f): """ Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request. """ # Check for WebSockets handshake is_websockets = ( - flow and - websockets.check_handshake(flow.request.headers) and - websockets.check_handshake(flow.response.headers) + f and + websockets.check_handshake(f.request.headers) and + websockets.check_handshake(f.response.headers) ) if is_websockets and not self.config.options.websockets: self.log( @@ -450,7 +451,7 @@ class HttpLayer(base.Layer): ) if is_websockets and self.config.options.websockets: - layer = pwebsockets.WebSocketsLayer(self, flow) + layer = pwebsockets.WebSocketsLayer(self, f) else: layer = self.ctx.next_layer(self) diff --git a/mitmproxy/protocol/http1.py b/mitmproxy/protocol/http1.py index 46839c58b..c00848042 100644 --- a/mitmproxy/protocol/http1.py +++ b/mitmproxy/protocol/http1.py @@ -1,16 +1,16 @@ -from mitmproxy import models -from mitmproxy.protocol import http +from mitmproxy import http +from mitmproxy.protocol import http as httpbase from netlib.http import http1 -class Http1Layer(http._HttpTransmissionLayer): +class Http1Layer(httpbase._HttpTransmissionLayer): def __init__(self, ctx, mode): super().__init__(ctx) self.mode = mode def read_request_headers(self): - return models.HTTPRequest.wrap( + return http.HTTPRequest.wrap( http1.read_request_head(self.client_conn.rfile) ) @@ -28,7 +28,7 @@ class Http1Layer(http._HttpTransmissionLayer): def read_response_headers(self): resp = http1.read_response_head(self.server_conn.rfile) - return models.HTTPResponse.wrap(resp) + return http.HTTPResponse.wrap(resp) def read_response_body(self, request, response): expected_size = http1.expected_http_body_size(request, response) @@ -68,5 +68,5 @@ class Http1Layer(http._HttpTransmissionLayer): return close_connection def __call__(self): - layer = http.HttpLayer(self, self.mode) + layer = httpbase.HttpLayer(self, self.mode) layer() diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 04d2b6081..a3ec03f4e 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -10,9 +10,9 @@ import queue import netlib.exceptions from mitmproxy import exceptions -from mitmproxy import models +from mitmproxy import http from mitmproxy.protocol import base -from mitmproxy.protocol import http +from mitmproxy.protocol import http as httpbase import netlib.http from netlib import tcp from netlib import basethread @@ -358,7 +358,7 @@ def detect_zombie_stream(func): return wrapper -class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread): +class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThread): def __init__(self, ctx, h2_connection, stream_id, request_headers): super().__init__( @@ -451,7 +451,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) self.request_arrived.wait() self.raise_zombie() first_line_format, method, scheme, host, port, path = http2.parse_headers(self.request_headers) - return models.HTTPRequest( + return http.HTTPRequest( first_line_format, method, scheme, @@ -547,7 +547,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) headers = self.response_headers.copy() headers.pop(":status", None) - return models.HTTPResponse( + return http.HTTPResponse( http_version=b"HTTP/2.0", status_code=status_code, reason=b'', @@ -597,7 +597,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) raise EnvironmentError('Http2SingleStreamLayer must be run as thread') def run(self): - layer = http.HttpLayer(self, self.mode) + layer = httpbase.HttpLayer(self, self.mode) try: layer() diff --git a/mitmproxy/protocol/http_replay.py b/mitmproxy/protocol/http_replay.py index 6f9340b5c..952c18176 100644 --- a/mitmproxy/protocol/http_replay.py +++ b/mitmproxy/protocol/http_replay.py @@ -1,9 +1,12 @@ import traceback import netlib.exceptions +from mitmproxy import log from mitmproxy import controller from mitmproxy import exceptions -from mitmproxy import models +from mitmproxy import http +from mitmproxy import flow +from mitmproxy import connections from netlib.http import http1 from netlib import basethread @@ -14,42 +17,42 @@ from netlib import basethread class RequestReplayThread(basethread.BaseThread): name = "RequestReplayThread" - def __init__(self, config, flow, event_queue, should_exit): + def __init__(self, config, f, event_queue, should_exit): """ event_queue can be a queue or None, if no scripthooks should be processed. """ - self.config, self.flow = config, flow - flow.live = True + self.config, self.f = config, f + f.live = True if event_queue: self.channel = controller.Channel(event_queue, should_exit) else: self.channel = None super().__init__( - "RequestReplay (%s)" % flow.request.url + "RequestReplay (%s)" % f.request.url ) def run(self): - r = self.flow.request + r = self.f.request first_line_format_backup = r.first_line_format server = None try: - self.flow.response = None + self.f.response = None # If we have a channel, run script hooks. if self.channel: - request_reply = self.channel.ask("request", self.flow) - if isinstance(request_reply, models.HTTPResponse): - self.flow.response = request_reply + request_reply = self.channel.ask("request", self.f) + if isinstance(request_reply, http.HTTPResponse): + self.f.response = request_reply - if not self.flow.response: + if not self.f.response: # In all modes, we directly connect to the server displayed if self.config.options.mode == "upstream": server_address = self.config.upstream_server.address - server = models.ServerConnection(server_address, (self.config.options.listen_host, 0)) + server = connections.ServerConnection(server_address, (self.config.options.listen_host, 0)) server.connect() if r.scheme == "https": - connect_request = models.make_connect_request((r.data.host, r.port)) + connect_request = http.make_connect_request((r.data.host, r.port)) server.wfile.write(http1.assemble_request(connect_request)) server.wfile.flush() resp = http1.read_response( @@ -61,46 +64,57 @@ class RequestReplayThread(basethread.BaseThread): raise exceptions.ReplayException("Upstream server refuses CONNECT request") server.establish_ssl( self.config.clientcerts, - sni=self.flow.server_conn.sni + sni=self.f.server_conn.sni ) r.first_line_format = "relative" else: r.first_line_format = "absolute" else: server_address = (r.host, r.port) - server = models.ServerConnection(server_address, (self.config.options.listen_host, 0)) + server = connections.ServerConnection( + server_address, + (self.config.options.listen_host, 0) + ) server.connect() if r.scheme == "https": server.establish_ssl( self.config.clientcerts, - sni=self.flow.server_conn.sni + sni=self.f.server_conn.sni ) r.first_line_format = "relative" server.wfile.write(http1.assemble_request(r)) server.wfile.flush() - self.flow.server_conn = server - self.flow.response = models.HTTPResponse.wrap(http1.read_response( - server.rfile, - r, - body_size_limit=self.config.options.body_size_limit - )) + self.f.server_conn = server + self.f.response = http.HTTPResponse.wrap( + http1.read_response( + server.rfile, + r, + body_size_limit=self.config.options.body_size_limit + ) + ) if self.channel: - response_reply = self.channel.ask("response", self.flow) + response_reply = self.channel.ask("response", self.f) if response_reply == exceptions.Kill: raise exceptions.Kill() except (exceptions.ReplayException, netlib.exceptions.NetlibException) as e: - self.flow.error = models.Error(str(e)) + self.f.error = flow.Error(str(e)) if self.channel: - self.channel.ask("error", self.flow) + self.channel.ask("error", self.f) except exceptions.Kill: # Kill should only be raised if there's a channel in the # first place. - self.channel.tell("log", controller.LogEntry("Connection killed", "info")) + self.channel.tell( + "log", + log.LogEntry("Connection killed", "info") + ) except Exception: - self.channel.tell("log", controller.LogEntry(traceback.format_exc(), "error")) + self.channel.tell( + "log", + log.LogEntry(traceback.format_exc(), "error") + ) finally: r.first_line_format = first_line_format_backup - self.flow.live = False + self.f.live = False if server: server.finish() diff --git a/mitmproxy/protocol/rawtcp.py b/mitmproxy/protocol/rawtcp.py index de9f3c246..3bd7b162e 100644 --- a/mitmproxy/protocol/rawtcp.py +++ b/mitmproxy/protocol/rawtcp.py @@ -4,8 +4,8 @@ from OpenSSL import SSL import netlib.exceptions import netlib.tcp -from mitmproxy import models -from mitmproxy.models import tcp +from mitmproxy import tcp +from mitmproxy import flow from mitmproxy.protocol import base @@ -20,8 +20,8 @@ class RawTCPLayer(base.Layer): self.connect() if not self.ignore: - flow = models.TCPFlow(self.client_conn, self.server_conn, self) - self.channel.ask("tcp_start", flow) + f = tcp.TCPFlow(self.client_conn, self.server_conn, self) + self.channel.ask("tcp_start", f) buf = memoryview(bytearray(self.chunk_size)) @@ -52,14 +52,14 @@ class RawTCPLayer(base.Layer): tcp_message = tcp.TCPMessage(dst == server, buf[:size].tobytes()) if not self.ignore: - flow.messages.append(tcp_message) - self.channel.ask("tcp_message", flow) + f.messages.append(tcp_message) + self.channel.ask("tcp_message", f) dst.sendall(tcp_message.content) except (socket.error, netlib.exceptions.TcpException, SSL.Error) as e: if not self.ignore: - flow.error = models.Error("TCP connection closed unexpectedly: {}".format(repr(e))) - self.channel.tell("tcp_error", flow) + f.error = flow.Error("TCP connection closed unexpectedly: {}".format(repr(e))) + self.channel.tell("tcp_error", f) finally: if not self.ignore: - self.channel.tell("tcp_end", flow) + self.channel.tell("tcp_end", f) diff --git a/mitmproxy/protocol/tls.py b/mitmproxy/protocol/tls.py index 265e90869..b287bb51d 100644 --- a/mitmproxy/protocol/tls.py +++ b/mitmproxy/protocol/tls.py @@ -224,7 +224,7 @@ def get_client_hello(client_conn): Peek into the socket and read all records that contain the initial client hello message. client_conn: - The :py:class:`client connection `. + The :py:class:`client connection `. Returns: The raw handshake packet bytes, without TLS record header(s). @@ -281,7 +281,7 @@ class TlsClientHello: """ Peek into the connection, read the initial client hello and parse it to obtain ALPN values. client_conn: - The :py:class:`client connection `. + The :py:class:`client connection `. Returns: :py:class:`client hello `. """ diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index 8064f12df..fa4fae478 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -13,7 +13,7 @@ class RootContext: Attributes: client_conn: - The :py:class:`client connection `. + The :py:class:`client connection `. channel: A :py:class:`~mitmproxy.controller.Channel` to communicate with the FlowMaster. Provides :py:meth:`.ask() ` and diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index c2a6a5c3e..b876f9ce5 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -4,7 +4,8 @@ import traceback import netlib.exceptions from mitmproxy import exceptions -from mitmproxy import models +from mitmproxy import connections +from mitmproxy import http from mitmproxy import log from mitmproxy.proxy import modes from mitmproxy.proxy import root_context @@ -66,7 +67,7 @@ class ConnectionHandler: def __init__(self, client_conn, client_address, config, channel): self.config = config """@type: mitmproxy.proxy.config.ProxyConfig""" - self.client_conn = models.ClientConnection( + self.client_conn = connections.ClientConnection( client_conn, client_address, None) @@ -135,7 +136,7 @@ class ConnectionHandler: # we send an HTTP error response, which is both # understandable by HTTP clients and humans. try: - error_response = models.make_error_response(502, repr(e)) + error_response = http.make_error_response(502, repr(e)) self.client_conn.send(http1.assemble_response(error_response)) except netlib.exceptions.TcpException: pass diff --git a/mitmproxy/models/tcp.py b/mitmproxy/tcp.py similarity index 91% rename from mitmproxy/models/tcp.py rename to mitmproxy/tcp.py index ae30730e6..af54c9d41 100644 --- a/mitmproxy/models/tcp.py +++ b/mitmproxy/tcp.py @@ -3,7 +3,7 @@ import time from typing import List import netlib.basetypes -from mitmproxy.models.flow import Flow +from mitmproxy import flow class TCPMessage(netlib.basetypes.Serializable): @@ -34,7 +34,7 @@ class TCPMessage(netlib.basetypes.Serializable): ) -class TCPFlow(Flow): +class TCPFlow(flow.Flow): """ A TCPFlow is a simplified representation of a TCP session. @@ -44,7 +44,7 @@ class TCPFlow(Flow): super().__init__("tcp", client_conn, server_conn, live) self.messages = [] # type: List[TCPMessage] - _stateobject_attributes = Flow._stateobject_attributes.copy() + _stateobject_attributes = flow.Flow._stateobject_attributes.copy() _stateobject_attributes.update( messages=List[TCPMessage] ) diff --git a/mitmproxy/tools/__init__.py b/mitmproxy/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mitmproxy/cmdline.py b/mitmproxy/tools/cmdline.py similarity index 99% rename from mitmproxy/cmdline.py rename to mitmproxy/tools/cmdline.py index df0b0a2f7..ddc4a5cef 100644 --- a/mitmproxy/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -750,7 +750,7 @@ def common_options(parser): def mitmproxy(): - # Don't import mitmproxy.console for mitmdump, urwid is not available on all + # Don't import mitmproxy.tools.console for mitmdump, urwid is not available on all # platforms. from .console import palettes diff --git a/mitmproxy/tools/console/__init__.py b/mitmproxy/tools/console/__init__.py new file mode 100644 index 000000000..947258c92 --- /dev/null +++ b/mitmproxy/tools/console/__init__.py @@ -0,0 +1,4 @@ +from mitmproxy.tools.console import master + + +__all__ = ["master"] diff --git a/mitmproxy/console/common.py b/mitmproxy/tools/console/common.py similarity index 99% rename from mitmproxy/console/common.py rename to mitmproxy/tools/console/common.py index 91253668f..d10d93213 100644 --- a/mitmproxy/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -8,7 +8,7 @@ import urwid.util import netlib from mitmproxy import utils -from mitmproxy.console import signals +from mitmproxy.tools.console import signals from mitmproxy import export from netlib import human diff --git a/mitmproxy/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py similarity index 98% rename from mitmproxy/console/flowdetailview.py rename to mitmproxy/tools/console/flowdetailview.py index 5c0eaf791..f13f9a1df 100644 --- a/mitmproxy/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -1,6 +1,6 @@ import urwid -from mitmproxy.console import common, searchable +from mitmproxy.tools.console import common, searchable from netlib import human diff --git a/mitmproxy/console/flowlist.py b/mitmproxy/tools/console/flowlist.py similarity index 99% rename from mitmproxy/console/flowlist.py rename to mitmproxy/tools/console/flowlist.py index 09a5d027f..31624229c 100644 --- a/mitmproxy/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -2,8 +2,8 @@ import urwid import netlib.http.url from mitmproxy import exceptions -from mitmproxy.console import common -from mitmproxy.console import signals +from mitmproxy.tools.console import common +from mitmproxy.tools.console import signals from mitmproxy import export diff --git a/mitmproxy/console/flowview.py b/mitmproxy/tools/console/flowview.py similarity index 97% rename from mitmproxy/console/flowview.py rename to mitmproxy/tools/console/flowview.py index 19afcdbc9..64caf4742 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -7,14 +7,14 @@ from mitmproxy import exceptions from typing import Optional, Union # noqa from mitmproxy import contentviews -from mitmproxy import models +from mitmproxy import http from mitmproxy import utils -from mitmproxy.console import common -from mitmproxy.console import flowdetailview -from mitmproxy.console import grideditor -from mitmproxy.console import searchable -from mitmproxy.console import signals -from mitmproxy.console import tabs +from mitmproxy.tools.console import common +from mitmproxy.tools.console import flowdetailview +from mitmproxy.tools.console import grideditor +from mitmproxy.tools.console import searchable +from mitmproxy.tools.console import signals +from mitmproxy.tools.console import tabs from mitmproxy import export from netlib.http import Headers from netlib.http import status_codes @@ -100,7 +100,7 @@ footer = [ class FlowViewHeader(urwid.WidgetWrap): - def __init__(self, master: "mitmproxy.console.master.ConsoleMaster", f: models.HTTPFlow): + def __init__(self, master: "mitmproxy.console.master.ConsoleMaster", f: http.HTTPFlow): self.master = master self.flow = f self._w = common.format_flow( @@ -208,7 +208,7 @@ class FlowView(tabs.Tabs): if error: signals.add_log(error, "error") # Give hint that you have to tab for the response. - if description == "No content" and isinstance(message, models.HTTPRequest): + if description == "No content" and isinstance(message, http.HTTPRequest): description = "No request content (press tab to view response)" # If the users has a wide terminal, he gets fewer lines; this should not be an issue. @@ -373,7 +373,7 @@ class FlowView(tabs.Tabs): message = self.flow.request else: if not self.flow.response: - self.flow.response = models.HTTPResponse.make(200, b"") + self.flow.response = http.HTTPResponse.make(200, b"") message = self.flow.response self.flow.backup() @@ -500,7 +500,7 @@ class FlowView(tabs.Tabs): signals.flow_change.send(self, flow = self.flow) def keypress(self, size, key): - conn = None # type: Optional[Union[models.HTTPRequest, models.HTTPResponse]] + conn = None # type: Optional[Union[http.HTTPRequest, http.HTTPResponse]] if self.tab_offset == TAB_REQ: conn = self.flow.request elif self.tab_offset == TAB_RESP: diff --git a/mitmproxy/console/grideditor/__init__.py b/mitmproxy/tools/console/grideditor/__init__.py similarity index 100% rename from mitmproxy/console/grideditor/__init__.py rename to mitmproxy/tools/console/grideditor/__init__.py diff --git a/mitmproxy/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py similarity index 99% rename from mitmproxy/console/grideditor/base.py rename to mitmproxy/tools/console/grideditor/base.py index 7b687c3e0..4505bb979 100644 --- a/mitmproxy/console/grideditor/base.py +++ b/mitmproxy/tools/console/grideditor/base.py @@ -9,8 +9,8 @@ from typing import Sequence from typing import Tuple import urwid -from mitmproxy.console import common -from mitmproxy.console import signals +from mitmproxy.tools.console import common +from mitmproxy.tools.console import signals FOOTER = [ ('heading_key', "enter"), ":edit ", diff --git a/mitmproxy/console/grideditor/col_bytes.py b/mitmproxy/tools/console/grideditor/col_bytes.py similarity index 96% rename from mitmproxy/console/grideditor/col_bytes.py rename to mitmproxy/tools/console/grideditor/col_bytes.py index 8956de88a..c951ce44f 100644 --- a/mitmproxy/console/grideditor/col_bytes.py +++ b/mitmproxy/tools/console/grideditor/col_bytes.py @@ -2,8 +2,8 @@ import os from typing import Callable, Optional import urwid -from mitmproxy.console import signals -from mitmproxy.console.grideditor import base +from mitmproxy.tools.console import signals +from mitmproxy.tools.console.grideditor import base from netlib import strutils diff --git a/mitmproxy/console/grideditor/col_subgrid.py b/mitmproxy/tools/console/grideditor/col_subgrid.py similarity index 92% rename from mitmproxy/console/grideditor/col_subgrid.py rename to mitmproxy/tools/console/grideditor/col_subgrid.py index 8a08f8388..3147e63d0 100644 --- a/mitmproxy/console/grideditor/col_subgrid.py +++ b/mitmproxy/tools/console/grideditor/col_subgrid.py @@ -1,6 +1,6 @@ import urwid -from mitmproxy.console.grideditor import base -from mitmproxy.console import signals +from mitmproxy.tools.console.grideditor import base +from mitmproxy.tools.console import signals from netlib.http import cookies diff --git a/mitmproxy/console/grideditor/col_text.py b/mitmproxy/tools/console/grideditor/col_text.py similarity index 92% rename from mitmproxy/console/grideditor/col_text.py rename to mitmproxy/tools/console/grideditor/col_text.py index ae15374c8..2d5192ae7 100644 --- a/mitmproxy/console/grideditor/col_text.py +++ b/mitmproxy/tools/console/grideditor/col_text.py @@ -5,8 +5,8 @@ In a nutshell, text columns are actually a proxy class for byte columns, which just encode/decodes contents. """ -from mitmproxy.console import signals -from mitmproxy.console.grideditor import col_bytes +from mitmproxy.tools.console import signals +from mitmproxy.tools.console.grideditor import col_bytes class Column(col_bytes.Column): diff --git a/mitmproxy/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py similarity index 94% rename from mitmproxy/console/grideditor/editors.py rename to mitmproxy/tools/console/grideditor/editors.py index 512c2416f..64361af74 100644 --- a/mitmproxy/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -3,12 +3,12 @@ import urwid from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy.addons import script -from mitmproxy.console import common -from mitmproxy.console.grideditor import base -from mitmproxy.console.grideditor import col_bytes -from mitmproxy.console.grideditor import col_text -from mitmproxy.console.grideditor import col_subgrid -from mitmproxy.console import signals +from mitmproxy.tools.console import common +from mitmproxy.tools.console.grideditor import base +from mitmproxy.tools.console.grideditor import col_bytes +from mitmproxy.tools.console.grideditor import col_text +from mitmproxy.tools.console.grideditor import col_subgrid +from mitmproxy.tools.console import signals from netlib.http import user_agents diff --git a/mitmproxy/console/help.py b/mitmproxy/tools/console/help.py similarity index 97% rename from mitmproxy/console/help.py rename to mitmproxy/tools/console/help.py index 5ee03f366..752ebf008 100644 --- a/mitmproxy/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -3,8 +3,8 @@ import platform import urwid from mitmproxy import flowfilter -from mitmproxy.console import common -from mitmproxy.console import signals +from mitmproxy.tools.console import common +from mitmproxy.tools.console import signals from netlib import version diff --git a/mitmproxy/console/master.py b/mitmproxy/tools/console/master.py similarity index 97% rename from mitmproxy/console/master.py rename to mitmproxy/tools/console/master.py index 4921373fa..e80cb0af5 100644 --- a/mitmproxy/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -24,16 +24,16 @@ from mitmproxy import flowfilter from mitmproxy import utils from mitmproxy.addons import state import mitmproxy.options -from mitmproxy.console import flowlist -from mitmproxy.console import flowview -from mitmproxy.console import grideditor -from mitmproxy.console import help -from mitmproxy.console import options -from mitmproxy.console import palettepicker -from mitmproxy.console import palettes -from mitmproxy.console import signals -from mitmproxy.console import statusbar -from mitmproxy.console import window +from mitmproxy.tools.console import flowlist +from mitmproxy.tools.console import flowview +from mitmproxy.tools.console import grideditor +from mitmproxy.tools.console import help +from mitmproxy.tools.console import options +from mitmproxy.tools.console import palettepicker +from mitmproxy.tools.console import palettes +from mitmproxy.tools.console import signals +from mitmproxy.tools.console import statusbar +from mitmproxy.tools.console import window from mitmproxy.flowfilter import FMarked from netlib import tcp, strutils diff --git a/mitmproxy/console/options.py b/mitmproxy/tools/console/options.py similarity index 97% rename from mitmproxy/console/options.py rename to mitmproxy/tools/console/options.py index d0f647138..04a0d08c2 100644 --- a/mitmproxy/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -1,11 +1,11 @@ import urwid from mitmproxy import contentviews -from mitmproxy.console import common -from mitmproxy.console import grideditor -from mitmproxy.console import palettes -from mitmproxy.console import select -from mitmproxy.console import signals +from mitmproxy.tools.console import common +from mitmproxy.tools.console import grideditor +from mitmproxy.tools.console import palettes +from mitmproxy.tools.console import select +from mitmproxy.tools.console import signals footer = [ ('heading_key', "enter/space"), ":toggle ", diff --git a/mitmproxy/console/palettepicker.py b/mitmproxy/tools/console/palettepicker.py similarity index 92% rename from mitmproxy/console/palettepicker.py rename to mitmproxy/tools/console/palettepicker.py index b0840d8bb..4554a1b64 100644 --- a/mitmproxy/console/palettepicker.py +++ b/mitmproxy/tools/console/palettepicker.py @@ -1,9 +1,9 @@ import urwid -from mitmproxy.console import common -from mitmproxy.console import palettes -from mitmproxy.console import select -from mitmproxy.console import signals +from mitmproxy.tools.console import common +from mitmproxy.tools.console import palettes +from mitmproxy.tools.console import select +from mitmproxy.tools.console import signals footer = [ ('heading_key', "enter/space"), ":select", diff --git a/mitmproxy/console/palettes.py b/mitmproxy/tools/console/palettes.py similarity index 100% rename from mitmproxy/console/palettes.py rename to mitmproxy/tools/console/palettes.py diff --git a/mitmproxy/console/pathedit.py b/mitmproxy/tools/console/pathedit.py similarity index 100% rename from mitmproxy/console/pathedit.py rename to mitmproxy/tools/console/pathedit.py diff --git a/mitmproxy/console/searchable.py b/mitmproxy/tools/console/searchable.py similarity index 98% rename from mitmproxy/console/searchable.py rename to mitmproxy/tools/console/searchable.py index f87669080..650b75a05 100644 --- a/mitmproxy/console/searchable.py +++ b/mitmproxy/tools/console/searchable.py @@ -1,6 +1,6 @@ import urwid -from mitmproxy.console import signals +from mitmproxy.tools.console import signals class Highlight(urwid.AttrMap): diff --git a/mitmproxy/console/select.py b/mitmproxy/tools/console/select.py similarity index 98% rename from mitmproxy/console/select.py rename to mitmproxy/tools/console/select.py index 9f6a4cad8..a990dff87 100644 --- a/mitmproxy/console/select.py +++ b/mitmproxy/tools/console/select.py @@ -1,6 +1,6 @@ import urwid -from mitmproxy.console import common +from mitmproxy.tools.console import common class _OptionWidget(urwid.WidgetWrap): diff --git a/mitmproxy/console/signals.py b/mitmproxy/tools/console/signals.py similarity index 100% rename from mitmproxy/console/signals.py rename to mitmproxy/tools/console/signals.py diff --git a/mitmproxy/console/statusbar.py b/mitmproxy/tools/console/statusbar.py similarity index 98% rename from mitmproxy/console/statusbar.py rename to mitmproxy/tools/console/statusbar.py index e7944a9e8..99f73727b 100644 --- a/mitmproxy/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -3,9 +3,9 @@ import os.path import urwid import netlib.http.url -from mitmproxy.console import common -from mitmproxy.console import pathedit -from mitmproxy.console import signals +from mitmproxy.tools.console import common +from mitmproxy.tools.console import pathedit +from mitmproxy.tools.console import signals from netlib import human diff --git a/mitmproxy/console/tabs.py b/mitmproxy/tools/console/tabs.py similarity index 100% rename from mitmproxy/console/tabs.py rename to mitmproxy/tools/console/tabs.py diff --git a/mitmproxy/console/window.py b/mitmproxy/tools/console/window.py similarity index 98% rename from mitmproxy/console/window.py rename to mitmproxy/tools/console/window.py index b972208e7..b20d2e33b 100644 --- a/mitmproxy/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -1,6 +1,6 @@ import urwid -from mitmproxy.console import signals +from mitmproxy.tools.console import signals class Window(urwid.Frame): diff --git a/mitmproxy/dump.py b/mitmproxy/tools/dump.py similarity index 100% rename from mitmproxy/dump.py rename to mitmproxy/tools/dump.py diff --git a/mitmproxy/main.py b/mitmproxy/tools/main.py similarity index 96% rename from mitmproxy/main.py rename to mitmproxy/tools/main.py index 19ee50272..05ae780ee 100644 --- a/mitmproxy/main.py +++ b/mitmproxy/tools/main.py @@ -2,7 +2,7 @@ import os import signal import sys -from mitmproxy import cmdline +from mitmproxy.tools import cmdline from mitmproxy import exceptions from mitmproxy.proxy import config from mitmproxy.proxy import server @@ -47,7 +47,7 @@ def mitmproxy(args=None): # pragma: no cover print("Error: mitmproxy's console interface is not supported on Windows. " "You can run mitmdump or mitmweb instead.", file=sys.stderr) sys.exit(1) - from . import console + from mitmproxy.tools import console version_check.check_pyopenssl_version() assert_utf8_env() @@ -79,7 +79,7 @@ def mitmproxy(args=None): # pragma: no cover def mitmdump(args=None): # pragma: no cover - from . import dump + from mitmproxy.tools import dump version_check.check_pyopenssl_version() @@ -113,7 +113,7 @@ def mitmdump(args=None): # pragma: no cover def mitmweb(args=None): # pragma: no cover - from . import web + from mitmproxy.tools import web version_check.check_pyopenssl_version() diff --git a/mitmproxy/tools/web/__init__.py b/mitmproxy/tools/web/__init__.py new file mode 100644 index 000000000..c2252af39 --- /dev/null +++ b/mitmproxy/tools/web/__init__.py @@ -0,0 +1,2 @@ +from mitmproxy.tools.web import master +__all__ = ["master"] diff --git a/mitmproxy/web/app.py b/mitmproxy/tools/web/app.py similarity index 98% rename from mitmproxy/web/app.py rename to mitmproxy/tools/web/app.py index c45b32a96..89b421868 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -11,13 +11,14 @@ import tornado.web from io import BytesIO from mitmproxy import flowfilter -from mitmproxy import models +from mitmproxy import flow +from mitmproxy import http from mitmproxy import contentviews from mitmproxy import io from netlib import version -def convert_flow_to_json_dict(flow: models.Flow) -> dict: +def convert_flow_to_json_dict(flow: flow.Flow) -> dict: """ Remove flow message content and cert to save transmission space. @@ -34,7 +35,7 @@ def convert_flow_to_json_dict(flow: models.Flow) -> dict: if flow.error: f["error"] = flow.error.get_state() - if isinstance(flow, models.HTTPFlow): + if isinstance(flow, http.HTTPFlow): if flow.request: f["request"] = { "method": flow.request.method, @@ -119,7 +120,7 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler): return self.application.master.state @property - def master(self) -> "mitmproxy.web.master.WebMaster": + def master(self) -> "mitmproxy.tools.web.master.WebMaster": return self.application.master @property diff --git a/mitmproxy/web/master.py b/mitmproxy/tools/web/master.py similarity index 99% rename from mitmproxy/web/master.py rename to mitmproxy/tools/web/master.py index f07d28c8c..2bca45552 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -12,7 +12,7 @@ from mitmproxy import exceptions from mitmproxy.addons import state from mitmproxy import options from mitmproxy import master -from mitmproxy.web import app +from mitmproxy.tools.web import app from netlib.http import authentication diff --git a/mitmproxy/web/static/app.css b/mitmproxy/tools/web/static/app.css similarity index 100% rename from mitmproxy/web/static/app.css rename to mitmproxy/tools/web/static/app.css diff --git a/mitmproxy/web/static/app.js b/mitmproxy/tools/web/static/app.js similarity index 100% rename from mitmproxy/web/static/app.js rename to mitmproxy/tools/web/static/app.js diff --git a/mitmproxy/web/static/fonts/fontawesome-webfont.eot b/mitmproxy/tools/web/static/fonts/fontawesome-webfont.eot similarity index 100% rename from mitmproxy/web/static/fonts/fontawesome-webfont.eot rename to mitmproxy/tools/web/static/fonts/fontawesome-webfont.eot diff --git a/mitmproxy/web/static/fonts/fontawesome-webfont.svg b/mitmproxy/tools/web/static/fonts/fontawesome-webfont.svg similarity index 100% rename from mitmproxy/web/static/fonts/fontawesome-webfont.svg rename to mitmproxy/tools/web/static/fonts/fontawesome-webfont.svg diff --git a/mitmproxy/web/static/fonts/fontawesome-webfont.ttf b/mitmproxy/tools/web/static/fonts/fontawesome-webfont.ttf similarity index 100% rename from mitmproxy/web/static/fonts/fontawesome-webfont.ttf rename to mitmproxy/tools/web/static/fonts/fontawesome-webfont.ttf diff --git a/mitmproxy/web/static/fonts/fontawesome-webfont.woff b/mitmproxy/tools/web/static/fonts/fontawesome-webfont.woff similarity index 100% rename from mitmproxy/web/static/fonts/fontawesome-webfont.woff rename to mitmproxy/tools/web/static/fonts/fontawesome-webfont.woff diff --git a/mitmproxy/web/static/images/chrome-devtools/LICENSE b/mitmproxy/tools/web/static/images/chrome-devtools/LICENSE similarity index 100% rename from mitmproxy/web/static/images/chrome-devtools/LICENSE rename to mitmproxy/tools/web/static/images/chrome-devtools/LICENSE diff --git a/mitmproxy/web/static/images/chrome-devtools/resourceCSSIcon.png b/mitmproxy/tools/web/static/images/chrome-devtools/resourceCSSIcon.png similarity index 100% rename from mitmproxy/web/static/images/chrome-devtools/resourceCSSIcon.png rename to mitmproxy/tools/web/static/images/chrome-devtools/resourceCSSIcon.png diff --git a/mitmproxy/web/static/images/chrome-devtools/resourceDocumentIcon.png b/mitmproxy/tools/web/static/images/chrome-devtools/resourceDocumentIcon.png similarity index 100% rename from mitmproxy/web/static/images/chrome-devtools/resourceDocumentIcon.png rename to mitmproxy/tools/web/static/images/chrome-devtools/resourceDocumentIcon.png diff --git a/mitmproxy/web/static/images/chrome-devtools/resourceJSIcon.png b/mitmproxy/tools/web/static/images/chrome-devtools/resourceJSIcon.png similarity index 100% rename from mitmproxy/web/static/images/chrome-devtools/resourceJSIcon.png rename to mitmproxy/tools/web/static/images/chrome-devtools/resourceJSIcon.png diff --git a/mitmproxy/web/static/images/chrome-devtools/resourcePlainIcon.png b/mitmproxy/tools/web/static/images/chrome-devtools/resourcePlainIcon.png similarity index 100% rename from mitmproxy/web/static/images/chrome-devtools/resourcePlainIcon.png rename to mitmproxy/tools/web/static/images/chrome-devtools/resourcePlainIcon.png diff --git a/mitmproxy/web/static/images/favicon.ico b/mitmproxy/tools/web/static/images/favicon.ico similarity index 100% rename from mitmproxy/web/static/images/favicon.ico rename to mitmproxy/tools/web/static/images/favicon.ico diff --git a/mitmproxy/web/static/images/resourceExecutableIcon.png b/mitmproxy/tools/web/static/images/resourceExecutableIcon.png similarity index 100% rename from mitmproxy/web/static/images/resourceExecutableIcon.png rename to mitmproxy/tools/web/static/images/resourceExecutableIcon.png diff --git a/mitmproxy/web/static/images/resourceFlashIcon.png b/mitmproxy/tools/web/static/images/resourceFlashIcon.png similarity index 100% rename from mitmproxy/web/static/images/resourceFlashIcon.png rename to mitmproxy/tools/web/static/images/resourceFlashIcon.png diff --git a/mitmproxy/web/static/images/resourceImageIcon.png b/mitmproxy/tools/web/static/images/resourceImageIcon.png similarity index 100% rename from mitmproxy/web/static/images/resourceImageIcon.png rename to mitmproxy/tools/web/static/images/resourceImageIcon.png diff --git a/mitmproxy/web/static/images/resourceJavaIcon.png b/mitmproxy/tools/web/static/images/resourceJavaIcon.png similarity index 100% rename from mitmproxy/web/static/images/resourceJavaIcon.png rename to mitmproxy/tools/web/static/images/resourceJavaIcon.png diff --git a/mitmproxy/web/static/images/resourceNotModifiedIcon.png b/mitmproxy/tools/web/static/images/resourceNotModifiedIcon.png similarity index 100% rename from mitmproxy/web/static/images/resourceNotModifiedIcon.png rename to mitmproxy/tools/web/static/images/resourceNotModifiedIcon.png diff --git a/mitmproxy/web/static/images/resourceRedirectIcon.png b/mitmproxy/tools/web/static/images/resourceRedirectIcon.png similarity index 100% rename from mitmproxy/web/static/images/resourceRedirectIcon.png rename to mitmproxy/tools/web/static/images/resourceRedirectIcon.png diff --git a/mitmproxy/web/static/vendor.css b/mitmproxy/tools/web/static/vendor.css similarity index 100% rename from mitmproxy/web/static/vendor.css rename to mitmproxy/tools/web/static/vendor.css diff --git a/mitmproxy/web/static/vendor.js b/mitmproxy/tools/web/static/vendor.js similarity index 100% rename from mitmproxy/web/static/vendor.js rename to mitmproxy/tools/web/static/vendor.js diff --git a/mitmproxy/web/templates/index.html b/mitmproxy/tools/web/templates/index.html similarity index 100% rename from mitmproxy/web/templates/index.html rename to mitmproxy/tools/web/templates/index.html diff --git a/mitmproxy/web/__init__.py b/mitmproxy/web/__init__.py deleted file mode 100644 index b21c894f6..000000000 --- a/mitmproxy/web/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from mitmproxy.web import master -__all__ = ["master"] diff --git a/setup.py b/setup.py index 9d85e545a..f856ff7ae 100644 --- a/setup.py +++ b/setup.py @@ -49,9 +49,9 @@ setup( include_package_data=True, entry_points={ 'console_scripts': [ - "mitmproxy = mitmproxy.main:mitmproxy", - "mitmdump = mitmproxy.main:mitmdump", - "mitmweb = mitmproxy.main:mitmweb", + "mitmproxy = mitmproxy.tools.main:mitmproxy", + "mitmdump = mitmproxy.tools.main:mitmdump", + "mitmweb = mitmproxy.tools.main:mitmweb", "pathod = pathod.pathod_cmdline:go_pathod", "pathoc = pathod.pathoc_cmdline:go_pathoc" ] diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index ce2d2de9a..e49f91bc1 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -4,8 +4,8 @@ from .. import tutils, mastertest from mitmproxy.addons import dumper from mitmproxy import exceptions -from mitmproxy import dump -from mitmproxy import models +from mitmproxy.tools import dump +from mitmproxy import http from mitmproxy import proxy import netlib.tutils import mock @@ -60,7 +60,7 @@ class TestDumper(mastertest.MasterTest): d.configure(dump.Options(tfile = sio), updated) flow = tutils.tflow() flow.request.content = None - flow.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) + flow.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) flow.response.content = None d.response(flow) assert "content missing" in sio.getvalue() diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index a9f18a51a..1045ec298 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -3,7 +3,7 @@ import io from mitmproxy.addons import termlog from mitmproxy import log -from mitmproxy import dump +from mitmproxy.tools import dump class TestTermLog(mastertest.MasterTest): diff --git a/test/mitmproxy/console/test_common.py b/test/mitmproxy/console/test_common.py index ac4e02091..1a12d3601 100644 --- a/test/mitmproxy/console/test_common.py +++ b/test/mitmproxy/console/test_common.py @@ -1,4 +1,4 @@ -import mitmproxy.console.common as common +from mitmproxy.tools.console import common from .. import tutils diff --git a/test/mitmproxy/console/test_help.py b/test/mitmproxy/console/test_help.py index 3c1cc57cc..d34f3f5d6 100644 --- a/test/mitmproxy/console/test_help.py +++ b/test/mitmproxy/console/test_help.py @@ -1,4 +1,4 @@ -import mitmproxy.console.help as help +import mitmproxy.tools.console.help as help from .. import tutils diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index 46eec9817..da29a2b1c 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -1,9 +1,9 @@ import gc import netlib.tutils -from mitmproxy import console +from mitmproxy.tools import console from mitmproxy import proxy -from mitmproxy.console import common +from mitmproxy.tools.console import common from .. import tutils, mastertest diff --git a/test/mitmproxy/console/test_palettes.py b/test/mitmproxy/console/test_palettes.py index a99730fa0..7490bae08 100644 --- a/test/mitmproxy/console/test_palettes.py +++ b/test/mitmproxy/console/test_palettes.py @@ -1,4 +1,4 @@ -import mitmproxy.console.palettes as palettes +import mitmproxy.tools.console.palettes as palettes from .. import tutils diff --git a/test/mitmproxy/console/test_pathedit.py b/test/mitmproxy/console/test_pathedit.py index 45c69a385..29f8de17c 100644 --- a/test/mitmproxy/console/test_pathedit.py +++ b/test/mitmproxy/console/test_pathedit.py @@ -1,6 +1,6 @@ import os from os.path import normpath -from mitmproxy.console import pathedit +from mitmproxy.tools.console import pathedit from mock import patch diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index f82eec404..0b57c2d48 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -4,7 +4,10 @@ from . import tutils import netlib.tutils from mitmproxy import master -from mitmproxy import io, proxy, models, options +from mitmproxy import io +from mitmproxy import proxy +from mitmproxy import http +from mitmproxy import options class TestMaster: @@ -19,7 +22,7 @@ class MasterTest: master.serverconnect(f.server_conn) master.request(f) if not f.error: - f.response = models.HTTPResponse.wrap( + f.response = http.HTTPResponse.wrap( netlib.tutils.tresp(content=content) ) master.response(f) diff --git a/test/mitmproxy/test_cmdline.py b/test/mitmproxy/test_cmdline.py index 55627408e..d664430a3 100644 --- a/test/mitmproxy/test_cmdline.py +++ b/test/mitmproxy/test_cmdline.py @@ -1,5 +1,5 @@ import argparse -from mitmproxy import cmdline +from mitmproxy.tools import cmdline from . import tutils diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index c062a0688..33d2423cf 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -2,7 +2,9 @@ import os import io import mitmproxy.io -from mitmproxy import dump, exceptions, proxy +from mitmproxy.tools import dump +from mitmproxy import exceptions +from mitmproxy import proxy from . import tutils, mastertest diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index d33d395b5..9ebcca344 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -8,14 +8,11 @@ from mitmproxy import flowfilter, options from mitmproxy.addons import state from mitmproxy.contrib import tnetstring from mitmproxy.exceptions import FlowReadException, Kill -from mitmproxy.models import Error -from mitmproxy.models import Flow -from mitmproxy.models import HTTPFlow -from mitmproxy.models import HTTPRequest -from mitmproxy.models import HTTPResponse +from mitmproxy import flow +from mitmproxy import http +from mitmproxy import connections from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer -from mitmproxy.models.connections import ClientConnection from mitmproxy import master from . import tutils @@ -62,7 +59,7 @@ class TestHTTPFlow: def test_backup(self): f = tutils.tflow() - f.response = HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) f.request.content = b"foo" assert not f.modified() f.backup() @@ -81,20 +78,20 @@ class TestHTTPFlow: def test_getset_state(self): f = tutils.tflow(resp=True) state = f.get_state() - assert f.get_state() == HTTPFlow.from_state( + assert f.get_state() == http.HTTPFlow.from_state( state).get_state() f.response = None - f.error = Error("error") + f.error = flow.Error("error") state = f.get_state() - assert f.get_state() == HTTPFlow.from_state( + assert f.get_state() == http.HTTPFlow.from_state( state).get_state() f2 = f.copy() f2.id = f.id # copy creates a different uuid assert f.get_state() == f2.get_state() assert not f == f2 - f2.error = Error("e2") + f2.error = flow.Error("e2") assert not f == f2 f.set_state(f2.get_state()) assert f.get_state() == f2.get_state() @@ -215,7 +212,7 @@ class TestState: assert c.add_flow(newf) assert c.active_flow_count() == 2 - f.response = HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(f) assert c.flow_count() == 2 assert c.active_flow_count() == 1 @@ -223,7 +220,7 @@ class TestState: assert not c.update_flow(None) assert c.active_flow_count() == 1 - newf.response = HTTPResponse.wrap(netlib.tutils.tresp()) + newf.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(newf) assert c.active_flow_count() == 0 @@ -231,7 +228,7 @@ class TestState: c = state.State() f = tutils.tflow() c.add_flow(f) - f.error = Error("message") + f.error = flow.Error("message") assert c.update_flow(f) c = state.State() @@ -255,7 +252,7 @@ class TestState: c.set_view_filter("~s") assert c.filter_txt == "~s" assert len(c.view) == 0 - f.response = HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) c.update_flow(f) assert len(c.view) == 1 c.set_view_filter(None) @@ -287,7 +284,7 @@ class TestState: def _add_response(self, state): f = tutils.tflow() state.add_flow(f) - f.response = HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) state.update_flow(f) def _add_error(self, state): @@ -316,7 +313,7 @@ class TestState: c.clear() c.load_flows(flows) - assert isinstance(c.flows[0], Flow) + assert isinstance(c.flows[0], flow.Flow) def test_accept_all(self): c = state.State() @@ -447,17 +444,17 @@ class TestFlowMaster: fm.addons.add(s) f = tutils.tflow(req=None) fm.clientconnect(f.client_conn) - f.request = HTTPRequest.wrap(netlib.tutils.treq()) + f.request = http.HTTPRequest.wrap(netlib.tutils.treq()) fm.request(f) assert s.flow_count() == 1 - f.response = HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) fm.response(f) assert s.flow_count() == 1 fm.clientdisconnect(f.client_conn) - f.error = Error("msg") + f.error = flow.Error("msg") fm.error(f) fm.shutdown() @@ -476,7 +473,7 @@ class TestRequest: assert r.get_state() == r2.get_state() def test_get_url(self): - r = HTTPRequest.wrap(netlib.tutils.treq()) + r = http.HTTPRequest.wrap(netlib.tutils.treq()) assert r.url == "http://address:22/path" @@ -497,7 +494,7 @@ class TestRequest: assert r.pretty_url == "https://foo.com:22/path" def test_replace(self): - r = HTTPRequest.wrap(netlib.tutils.treq()) + r = http.HTTPRequest.wrap(netlib.tutils.treq()) r.path = "path/foo" r.headers["Foo"] = "fOo" r.content = b"afoob" @@ -507,7 +504,7 @@ class TestRequest: assert r.headers["boo"] == "boo" def test_constrain_encoding(self): - r = HTTPRequest.wrap(netlib.tutils.treq()) + r = http.HTTPRequest.wrap(netlib.tutils.treq()) r.headers["accept-encoding"] = "gzip, oink" r.constrain_encoding() assert "oink" not in r.headers["accept-encoding"] @@ -517,7 +514,7 @@ class TestRequest: assert "oink" not in r.headers["accept-encoding"] def test_get_content_type(self): - resp = HTTPResponse.wrap(netlib.tutils.tresp()) + resp = http.HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = Headers(content_type="text/plain") assert resp.headers["content-type"] == "text/plain" @@ -531,7 +528,7 @@ class TestResponse: assert resp2.get_state() == resp.get_state() def test_replace(self): - r = HTTPResponse.wrap(netlib.tutils.tresp()) + r = http.HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["Foo"] = "fOo" r.content = b"afoob" assert r.replace("foo(?i)", "boo") == 3 @@ -539,7 +536,7 @@ class TestResponse: assert r.headers["boo"] == "boo" def test_get_content_type(self): - resp = HTTPResponse.wrap(netlib.tutils.tresp()) + resp = http.HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = Headers(content_type="text/plain") assert resp.headers["content-type"] == "text/plain" @@ -547,13 +544,13 @@ class TestResponse: class TestError: def test_getset_state(self): - e = Error("Error") + e = flow.Error("Error") state = e.get_state() - assert Error.from_state(state).get_state() == e.get_state() + assert flow.Error.from_state(state).get_state() == e.get_state() assert e.copy() - e2 = Error("bar") + e2 = flow.Error("bar") assert not e == e2 e.set_state(e2.get_state()) assert e.get_state() == e2.get_state() @@ -562,14 +559,14 @@ class TestError: assert e3.get_state() == e.get_state() def test_repr(self): - e = Error("yay") + e = flow.Error("yay") assert repr(e) class TestClientConnection: def test_state(self): c = tutils.tclient_conn() - assert ClientConnection.from_state(c.get_state()).get_state() == \ + assert connections.ClientConnection.from_state(c.get_state()).get_state() == \ c.get_state() c2 = tutils.tclient_conn() diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 5904ce5b3..7d4011844 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -2,10 +2,10 @@ import os import mock from OpenSSL import SSL -from mitmproxy import cmdline +from mitmproxy.tools import cmdline from mitmproxy import options from mitmproxy.proxy import ProxyConfig -from mitmproxy.models.connections import ServerConnection +from mitmproxy import connections from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler from mitmproxy.proxy import config from netlib.exceptions import TcpDisconnect @@ -18,7 +18,7 @@ class TestServerConnection: def test_simple(self): self.d = test.Daemon() - sc = ServerConnection((self.d.IFACE, self.d.port)) + sc = connections.ServerConnection((self.d.IFACE, self.d.port)) sc.connect() f = tutils.tflow() f.server_conn = sc @@ -36,7 +36,7 @@ class TestServerConnection: def test_terminate_error(self): self.d = test.Daemon() - sc = ServerConnection((self.d.IFACE, self.d.port)) + sc = connections.ServerConnection((self.d.IFACE, self.d.port)) sc.connect() sc.connection = mock.Mock() sc.connection.recv = mock.Mock(return_value=False) diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 7616b5a78..93a82954a 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -6,9 +6,10 @@ import netlib.tutils from mitmproxy import controller from mitmproxy import options from mitmproxy.addons import script -from mitmproxy.models import HTTPResponse, HTTPFlow +from mitmproxy import http from mitmproxy.proxy.config import HostMatcher, parse_server_spec -from netlib import tcp, http, socks +import netlib.http +from netlib import tcp, socks from netlib.certutils import SSLCert from netlib.exceptions import HttpReadDisconnect, HttpException from netlib.http import authentication, http1 @@ -183,9 +184,9 @@ class TcpMixin: assert n.status_code == 304 assert i.status_code == 305 assert i2.status_code == 306 - assert any(f.response.status_code == 304 for f in self.master.state.flows if isinstance(f, HTTPFlow)) - assert not any(f.response.status_code == 305 for f in self.master.state.flows if isinstance(f, HTTPFlow)) - assert not any(f.response.status_code == 306 for f in self.master.state.flows if isinstance(f, HTTPFlow)) + assert any(f.response.status_code == 304 for f in self.master.state.flows if isinstance(f, http.HTTPFlow)) + assert not any(f.response.status_code == 305 for f in self.master.state.flows if isinstance(f, http.HTTPFlow)) + assert not any(f.response.status_code == 306 for f in self.master.state.flows if isinstance(f, http.HTTPFlow)) # Test that we get the original SSL cert if self.ssl: @@ -293,7 +294,7 @@ class TestHTTPAuth(tservers.HTTPProxyTest): h'%s'='%s' """ % ( self.server.port, - http.authentication.BasicProxyAuth.AUTH_HEADER, + netlib.http.authentication.BasicProxyAuth.AUTH_HEADER, authentication.assemble_http_basic_auth("basic", "test", "test") )) assert ret.status_code == 202 @@ -310,7 +311,7 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest): '/p/202' h'%s'='%s' """ % ( - http.authentication.BasicWebsiteAuth.AUTH_HEADER, + netlib.http.authentication.BasicWebsiteAuth.AUTH_HEADER, authentication.assemble_http_basic_auth("basic", "test", "test") )) assert ret.status_code == 202 @@ -790,7 +791,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): class MasterFakeResponse(tservers.TestMaster): @controller.handler def request(self, f): - f.response = HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = http.HTTPResponse.wrap(netlib.tutils.tresp()) class TestFakeResponse(tservers.HTTPProxyTest): @@ -869,7 +870,7 @@ class MasterIncomplete(tservers.TestMaster): @controller.handler def request(self, f): - resp = HTTPResponse.wrap(netlib.tutils.tresp()) + resp = http.HTTPResponse.wrap(netlib.tutils.tresp()) resp.content = None f.response = resp diff --git a/test/mitmproxy/test_utils.py b/test/mitmproxy/test_utils.py index 79b72eece..ed59f4844 100644 --- a/test/mitmproxy/test_utils.py +++ b/test/mitmproxy/test_utils.py @@ -5,7 +5,7 @@ utils.CERT_SLEEP_TIME = 0 def test_pkg_data(): - assert utils.pkg_data.path("console") + assert utils.pkg_data.path("tools/console") tutils.raises("does not exist", utils.pkg_data.path, "nonexistent") diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py index 89354960b..8fc3378a2 100644 --- a/test/mitmproxy/test_web_app.py +++ b/test/mitmproxy/test_web_app.py @@ -1,8 +1,8 @@ import tornado.testing from mitmproxy import proxy -from mitmproxy.web import app -from mitmproxy.web import master as webmaster +from mitmproxy.tools.web import app +from mitmproxy.tools.web import master as webmaster class TestApp(tornado.testing.AsyncHTTPTestCase): diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index 267715ade..77280644b 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -1,4 +1,4 @@ -from mitmproxy.web import master +from mitmproxy.tools.web import master from mitmproxy import proxy from . import mastertest diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index 2869a5f50..9724516b2 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -12,11 +12,10 @@ import io import netlib.utils import netlib.tutils from mitmproxy import controller -from mitmproxy.models import ( - ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow -) -from mitmproxy.models.tcp import TCPMessage -from mitmproxy.models.flow import Flow +from mitmproxy import connections +from mitmproxy import flow +from mitmproxy import http +from mitmproxy import tcp def _skip_windows(*args): @@ -48,7 +47,7 @@ def skip_appveyor(fn): return fn -class DummyFlow(Flow): +class DummyFlow(flow.Flow): """A flow that is neither HTTP nor TCP.""" def __init__(self, client_conn, server_conn, live=None): @@ -76,13 +75,13 @@ def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None): server_conn = tserver_conn() if messages is True: messages = [ - TCPMessage(True, b"hello"), - TCPMessage(False, b"it's me"), + tcp.TCPMessage(True, b"hello"), + tcp.TCPMessage(False, b"it's me"), ] if err is True: err = terr() - f = TCPFlow(client_conn, server_conn) + f = tcp.TCPFlow(client_conn, server_conn) f.messages = messages f.error = err f.reply = controller.DummyReply() @@ -110,11 +109,11 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): err = terr() if req: - req = HTTPRequest.wrap(req) + req = http.HTTPRequest.wrap(req) if resp: - resp = HTTPResponse.wrap(resp) + resp = http.HTTPResponse.wrap(resp) - f = HTTPFlow(client_conn, server_conn) + f = http.HTTPFlow(client_conn, server_conn) f.request = req f.response = resp f.error = err @@ -126,7 +125,7 @@ def tclient_conn(): """ @return: mitmproxy.proxy.connection.ClientConnection """ - c = ClientConnection.from_state(dict( + c = connections.ClientConnection.from_state(dict( address=dict(address=("address", 22), use_ipv6=True), clientcert=None, ssl_established=False, @@ -142,7 +141,7 @@ def tserver_conn(): """ @return: mitmproxy.proxy.connection.ServerConnection """ - c = ServerConnection.from_state(dict( + c = connections.ServerConnection.from_state(dict( address=dict(address=("address", 22), use_ipv6=True), source_address=dict(address=("address", 22), use_ipv6=True), ip_address=None, @@ -163,7 +162,7 @@ def terr(content="error"): """ @return: mitmproxy.protocol.primitives.Error """ - err = Error(content) + err = flow.Error(content) return err