Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2025-01-30 14:58:38 +00:00
Code Issues Projects Releases Wiki Activity

Rip out master handler mechanism

Browse Source 
All events are now handled by addons, and we no longer support any events on
master.
This commit is contained in:
Aldo Cortesi 2017-03-16 15:40:43 +13:00
parent 3de9829003
commit c5e0dc64b9
17 changed files with 116 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
mitmproxy/addonmanager.py
View File

@ -1,5 +1,6 @@
from mitmproxy import exceptions
from mitmproxy import eventsequence
from mitmproxy import controller
from . import ctx
import pprint
@ -60,6 +61,32 @@ class AddonManager:
def __str__(self):
return pprint.pformat([str(i) for i in self.chain])
def handle_lifecycle(self, name, message):
"""
Handle a lifecycle event.
"""
if not hasattr(message, "reply"): # pragma: no cover
raise exceptions.ControlException(
"Message %s has no reply attribute" % message
)
# We can use DummyReply objects multiple times. We only clear them up on
# the next handler so that we can access value and state in the
# meantime.
if isinstance(message.reply, controller.DummyReply):
message.reply.reset()
self.trigger(name, message)
if message.reply.state != "taken":
message.reply.take()
if not message.reply.has_message:
message.reply.ack()
message.reply.commit()
if isinstance(message.reply, controller.DummyReply):
message.reply.mark_reset()
def invoke_addon(self, addon, name, *args, **kwargs):
"""
Invoke an event on an addon. This method must run within an

112
mitmproxy/controller.py
View File

@ -1,4 +1,3 @@
import functools
import queue
from mitmproxy import exceptions
@ -14,8 +13,8 @@ class Channel:
def ask(self, mtype, m):
"""
Decorate a message with a reply attribute, and send it to the
master. Then wait for a response.
Decorate a message with a reply attribute, and send it to the master.
Then wait for a response.
Raises:
exceptions.Kill: All connections should be closed immediately.
@ -36,83 +35,42 @@ class Channel:
def tell(self, mtype, m):
"""
Decorate a message with a dummy reply attribute, send it to the
master, then return immediately.
Decorate a message with a dummy reply attribute, send it to the master,
then return immediately.
"""
m.reply = DummyReply()
self.q.put((mtype, m))
def handler(f):
@functools.wraps(f)
def wrapper(master, message):
if not hasattr(message, "reply"):
raise exceptions.ControlException("Message %s has no reply attribute" % message)
# DummyReplys may be reused multiple times.
# We only clear them up on the next handler so that we can access value and
# state in the meantime.
if isinstance(message.reply, DummyReply):
message.reply.reset()
# The following ensures that inheritance with wrapped handlers in the
# base class works. If we're the first handler, then responsibility for
# acking is ours. If not, it's someone else's and we ignore it.
handling = False
# We're the first handler - ack responsibility is ours
if message.reply.state == "unhandled":
handling = True
message.reply.handle()
with master.handlecontext():
ret = f(master, message)
master.addons.trigger(f.__name__, message)
# Reset the handled flag - it's common for us to feed the same object
# through handlers repeatedly, so we don't want this to persist across
# calls.
if handling and message.reply.state == "handled":
message.reply.take()
if not message.reply.has_message:
message.reply.ack()
message.reply.commit()
# DummyReplys may be reused multiple times.
if isinstance(message.reply, DummyReply):
message.reply.mark_reset()
return ret
# Mark this function as a handler wrapper
wrapper.__dict__["__handler"] = True
return wrapper
NO_REPLY = object() # special object we can distinguish from a valid "None" reply.
class Reply:
"""
Messages sent through a channel are decorated with a "reply" attribute.
This object is used to respond to the message through the return
channel.
Messages sent through a channel are decorated with a "reply" attribute. This
object is used to respond to the message through the return channel.
"""
def __init__(self, obj):
self.obj = obj
self.q = queue.Queue() # type: queue.Queue
self._state = "unhandled" # "unhandled" -> "handled" -> "taken" -> "committed"
self.value = NO_REPLY # holds the reply value. May change before things are actually commited.
self._state = "start" # "start" -> "taken" -> "committed"
# Holds the reply value. May change before things are actually commited.
self.value = NO_REPLY
@property
def state(self):
"""
The state the reply is currently in. A normal reply object goes sequentially through the following lifecycle:
The state the reply is currently in. A normal reply object goes
sequentially through the following lifecycle:
1. unhandled: Initial State.
2. handled: The reply object has been handled by the topmost handler function.
3. taken: The reply object has been taken to be commited.
4. committed: The reply has been sent back to the requesting party.
1. start: Initial State.
2. taken: The reply object has been taken to be commited.
3. committed: The reply has been sent back to the requesting party.
This attribute is read-only and can only be modified by calling one of state transition functions.
This attribute is read-only and can only be modified by calling one of
state transition functions.
"""
return self._state
@ -120,47 +78,43 @@ class Reply:
def has_message(self):
return self.value != NO_REPLY
def handle(self):
"""
Reply are handled by controller.handlers, which may be nested. The first handler takes
responsibility and handles the reply.
"""
if self.state != "unhandled":
raise exceptions.ControlException("Reply is {}, but expected it to be unhandled.".format(self.state))
self._state = "handled"
def take(self):
"""
Scripts or other parties make "take" a reply out of a normal flow.
For example, intercepted flows are taken out so that the connection thread does not proceed.
"""
if self.state != "handled":
raise exceptions.ControlException("Reply is {}, but expected it to be handled.".format(self.state))
if self.state != "start":
raise exceptions.ControlException(
"Reply is {}, but expected it to be start.".format(self.state)
)
self._state = "taken"
def commit(self):
"""
Ultimately, messages are commited. This is done either automatically by the handler
if the message is not taken or manually by the entity which called .take().
Ultimately, messages are commited. This is done either automatically by
if the message is not taken or manually by the entity which called
.take().
"""
if self.state != "taken":
raise exceptions.ControlException("Reply is {}, but expected it to be taken.".format(self.state))
raise exceptions.ControlException(
"Reply is {}, but expected it to be taken.".format(self.state)
)
if not self.has_message:
raise exceptions.ControlException("There is no reply message.")
self._state = "committed"
self.q.put(self.value)
def ack(self, force=False):
if self.state not in {"start", "taken"}:
raise exceptions.ControlException(
"Reply is {}, but expected it to be start or taken.".format(self.state)
)
self.send(self.obj, force)
def kill(self, force=False):
self.send(exceptions.Kill, force)
def send(self, msg, force=False):
if self.state not in ("handled", "taken"):
raise exceptions.ControlException(
"Reply is {}, did not expect a call to .send().".format(self.state)
)
if self.has_message and not force:
raise exceptions.ControlException("There is already a reply message.")
self.value = msg
@ -174,7 +128,7 @@ class Reply:
class DummyReply(Reply):
"""
A reply object that is not connected to anything. In contrast to regular
Reply objects, DummyReply objects are reset to "unhandled" at the end of an
Reply objects, DummyReply objects are reset to "start" at the end of an
handler so that they can be used multiple times. Useful when we need an
object to seem like it has a channel, and during testing.
"""
@ -189,7 +143,7 @@ class DummyReply(Reply):
def reset(self):
if self._should_reset:
self._state = "unhandled"
self._state = "start"
self.value = NO_REPLY
def __del__(self):

7
mitmproxy/flow.py
View File

@ -144,7 +144,7 @@ class Flow(stateobject.StateObject):
@property
def killable(self):
return self.reply and self.reply.state in {"handled", "taken"}
return self.reply and self.reply.state == "taken"
def kill(self):
"""
@ -152,8 +152,9 @@ class Flow(stateobject.StateObject):
"""
self.error = Error("Connection killed")
self.intercepted = False
# reply.state should only be "handled" or "taken" here.
# if none of this is the case, .take() will raise an exception.
# reply.state should be "taken" here, or .take() will raise an
# exception.
if self.reply.state != "taken":
self.reply.take()
self.reply.kill(force=True)

99
mitmproxy/master.py
View File

@ -93,18 +93,7 @@ class Master:
raise exceptions.ControlException(
"Unknown event %s" % repr(mtype)
)
handle_func = getattr(self, mtype)
if not callable(handle_func):
raise exceptions.ControlException(
"Handler %s not callable" % mtype
)
if not handle_func.__dict__.get("__handler"):
raise exceptions.ControlException(
"Handler function %s is not decorated with controller.handler" % (
handle_func
)
)
handle_func(obj)
self.addons.handle_lifecycle(mtype, obj)
self.event_queue.task_done()
changed = True
except queue.Empty:
@ -143,7 +132,7 @@ class Master:
f.request.scheme = self.server.config.upstream_server.scheme
f.reply = controller.DummyReply()
for e, o in eventsequence.iterate(f):
getattr(self, e)(o)
self.addons.handle_lifecycle(e, o)
def replay_request(
self,
@ -199,87 +188,3 @@ class Master:
if block:
rt.join()
return rt
@controller.handler
def log(self, l):
pass
@controller.handler
def clientconnect(self, root_layer):
pass
@controller.handler
def clientdisconnect(self, root_layer):
pass
@controller.handler
def serverconnect(self, server_conn):
pass
@controller.handler
def serverdisconnect(self, server_conn):
pass
@controller.handler
def next_layer(self, top_layer):
pass
@controller.handler
def http_connect(self, f):
pass
@controller.handler
def error(self, f):
pass
@controller.handler
def requestheaders(self, f):
pass
@controller.handler
def request(self, f):
pass
@controller.handler
def responseheaders(self, f):
pass
@controller.handler
def response(self, f):
pass
@controller.handler
def websocket_handshake(self, f):
pass
@controller.handler
def websocket_start(self, flow):
pass
@controller.handler
def websocket_message(self, flow):
pass
@controller.handler
def websocket_error(self, flow):
pass
@controller.handler
def websocket_end(self, flow):
pass
@controller.handler
def tcp_start(self, flow):
pass
@controller.handler
def tcp_message(self, flow):
pass
@controller.handler
def tcp_error(self, flow):
pass
@controller.handler
def tcp_end(self, flow):
pass

17
mitmproxy/test/taddons.py
View File

@ -3,29 +3,26 @@ import contextlib
import mitmproxy.master
import mitmproxy.options
from mitmproxy import proxy
from mitmproxy import addonmanager
from mitmproxy import eventsequence
class _AddonWrapper:
def __init__(self, master, addons):
self.master = master
self.addons = addons
class TestAddons(addonmanager.AddonManager):
def __init__(self, master):
super().__init__(master)
def trigger(self, event, *args, **kwargs):
if event == "log":
self.master.logs.append(args[0])
else:
self.master.events.append((event, args, kwargs))
return self.addons.trigger(event, *args, **kwargs)
def __getattr__(self, attr):
return getattr(self.addons, attr)
super().trigger(event, *args, **kwargs)
class RecordingMaster(mitmproxy.master.Master):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.addons = _AddonWrapper(self, self.addons)
self.addons = TestAddons(self)
self.events = []
self.logs = []
@ -76,7 +73,7 @@ class context:
Cycles the flow through the events for the flow. Stops if a reply
is taken (as in flow interception).
"""
f.reply._state = "handled"
f.reply._state = "start"
for evt, arg in eventsequence.iterate(f):
h = getattr(addon, evt, None)
if h:

1
test/mitmproxy/addons/test_disable_h2c.py
View File

@ -31,7 +31,6 @@ class TestDisableH2CleartextUpgrade:
b = io.BytesIO(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n")
f = tflow.tflow()
f.request = http.HTTPRequest.wrap(http1.read_request(b))
f.reply.handle()
f.intercept()
a.request(f)

1
test/mitmproxy/addons/test_intercept.py
View File

@ -29,6 +29,5 @@ def test_simple():
assert not f.intercepted
f = tflow.tflow(resp=True)
f.reply._state = "handled"
r.response(f)
assert f.intercepted

1
test/mitmproxy/proxy/test_server.py
View File

@ -5,7 +5,6 @@ import pytest
from unittest import mock
from mitmproxy.test import tutils
from mitmproxy import controller
from mitmproxy import options
from mitmproxy.addons import script
from mitmproxy.addons import proxyauth

25
test/mitmproxy/test_controller.py
View File

@ -60,7 +60,6 @@ class TestChannel:
def reply():
m, obj = q.get()
assert m == "test"
obj.reply.handle()
obj.reply.send(42)
obj.reply.take()
obj.reply.commit()
@ -82,10 +81,7 @@ class TestChannel:
class TestReply:
def test_simple(self):
reply = controller.Reply(42)
assert reply.state == "unhandled"
reply.handle()
assert reply.state == "handled"
assert reply.state == "start"
reply.send("foo")
assert reply.value == "foo"
@ -101,7 +97,6 @@ class TestReply:
def test_kill(self):
reply = controller.Reply(43)
reply.handle()
reply.kill()
reply.take()
reply.commit()
@ -109,7 +104,6 @@ class TestReply:
def test_ack(self):
reply = controller.Reply(44)
reply.handle()
reply.ack()
reply.take()
reply.commit()
@ -117,7 +111,6 @@ class TestReply:
def test_reply_none(self):
reply = controller.Reply(45)
reply.handle()
reply.send(None)
reply.take()
reply.commit()
@ -125,7 +118,6 @@ class TestReply:
def test_commit_no_reply(self):
reply = controller.Reply(46)
reply.handle()
reply.take()
with pytest.raises(ControlException):
reply.commit()
@ -134,7 +126,6 @@ class TestReply:
def test_double_send(self):
reply = controller.Reply(47)
reply.handle()
reply.send(1)
with pytest.raises(ControlException):
reply.send(2)
@ -142,12 +133,11 @@ class TestReply:
reply.commit()
def test_state_transitions(self):
states = {"unhandled", "handled", "taken", "committed"}
states = {"start", "taken", "committed"}
accept = {
"handle": {"unhandled"},
"take": {"handled"},
"take": {"start"},
"commit": {"taken"},
"ack": {"handled", "taken"},
"ack": {"start", "taken"},
}
for fn, ok in accept.items():
for state in states:
@ -166,7 +156,6 @@ class TestReply:
reply = controller.Reply(47)
with pytest.raises(ControlException):
reply.__del__()
reply.handle()
reply.ack()
reply.take()
reply.commit()
@ -176,24 +165,22 @@ class TestDummyReply:
def test_simple(self):
reply = controller.DummyReply()
for _ in range(2):
reply.handle()
reply.ack()
reply.take()
reply.commit()
reply.mark_reset()
reply.reset()
assert reply.state == "unhandled"
assert reply.state == "start"
def test_reset(self):
reply = controller.DummyReply()
reply.handle()
reply.ack()
reply.take()
reply.commit()
reply.mark_reset()
assert reply.state == "committed"
reply.reset()
assert reply.state == "unhandled"
assert reply.state == "start"
def test_del(self):
reply = controller.DummyReply()

22
test/mitmproxy/test_examples.py
View File

@ -41,7 +41,7 @@ class TestScripts(tservers.MasterTest):
def test_add_header(self):
m, _ = tscript("simple/add_header.py")
f = tflow.tflow(resp=tutils.tresp())
m.response(f)
m.addons.handle_lifecycle("response", f)
assert f.response.headers["newheader"] == "foo"
def test_custom_contentviews(self):
@ -56,7 +56,7 @@ class TestScripts(tservers.MasterTest):
m, sc = tscript("simple/modify_body_inject_iframe.py", "http://example.org/evil_iframe")
f = tflow.tflow(resp=tutils.tresp(content=b"<html><body>mitmproxy</body></html>"))
m.response(f)
m.addons.handle_lifecycle("response", f)
content = f.response.content
assert b'iframe' in content and b'evil_iframe' in content
@ -65,41 +65,41 @@ class TestScripts(tservers.MasterTest):
form_header = Headers(content_type="application/x-www-form-urlencoded")
f = tflow.tflow(req=tutils.treq(headers=form_header))
m.request(f)
m.addons.handle_lifecycle("request", f)
assert f.request.urlencoded_form["mitmproxy"] == "rocks"
f.request.headers["content-type"] = ""
m.request(f)
m.addons.handle_lifecycle("request", f)
assert list(f.request.urlencoded_form.items()) == [("foo", "bar")]
def test_modify_querystring(self):
m, sc = tscript("simple/modify_querystring.py")
f = tflow.tflow(req=tutils.treq(path="/search?q=term"))
m.request(f)
m.addons.handle_lifecycle("request", f)
assert f.request.query["mitmproxy"] == "rocks"
f.request.path = "/"
m.request(f)
m.addons.handle_lifecycle("request", f)
assert f.request.query["mitmproxy"] == "rocks"
def test_arguments(self):
m, sc = tscript("simple/script_arguments.py", "mitmproxy rocks")
f = tflow.tflow(resp=tutils.tresp(content=b"I <3 mitmproxy"))
m.response(f)
m.addons.handle_lifecycle("response", f)
assert f.response.content == b"I <3 rocks"
def test_redirect_requests(self):
m, sc = tscript("simple/redirect_requests.py")
f = tflow.tflow(req=tutils.treq(host="example.org"))
m.request(f)
m.addons.handle_lifecycle("request", f)
assert f.request.host == "mitmproxy.org"
def test_send_reply_from_proxy(self):
m, sc = tscript("simple/send_reply_from_proxy.py")
f = tflow.tflow(req=tutils.treq(host="example.com", port=80))
m.request(f)
m.addons.handle_lifecycle("request", f)
assert f.response.content == b"Hello World"
def test_dns_spoofing(self):
@ -109,13 +109,13 @@ class TestScripts(tservers.MasterTest):
host_header = Headers(host=original_host)
f = tflow.tflow(req=tutils.treq(headers=host_header, port=80))
m.requestheaders(f)
m.addons.handle_lifecycle("requestheaders", f)
# Rewrite by reverse proxy mode
f.request.scheme = "https"
f.request.port = 443
m.request(f)
m.addons.handle_lifecycle("request", f)
assert f.request.scheme == "http"
assert f.request.port == 80

10
test/mitmproxy/test_flow.py
View File

@ -122,19 +122,19 @@ class TestFlowMaster:
fm = master.Master(None, DummyServer())
fm.addons.add(s)
f = tflow.tflow(req=None)
fm.clientconnect(f.client_conn)
fm.addons.handle_lifecycle("clientconnect", f.client_conn)
f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
fm.request(f)
fm.addons.handle_lifecycle("request", f)
assert len(s.flows) == 1
f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
fm.response(f)
fm.addons.handle_lifecycle("response", f)
assert len(s.flows) == 1
fm.clientdisconnect(f.client_conn)
fm.addons.handle_lifecycle("clientdisconnect", f.client_conn)
f.error = flow.Error("msg")
fm.error(f)
fm.addons.handle_lifecycle("error", f)
fm.shutdown()

2
test/mitmproxy/test_http.py
View File

@ -175,7 +175,6 @@ class TestHTTPFlow:
def test_kill(self):
f = tflow.tflow()
f.reply.handle()
f.intercept()
assert f.killable
f.kill()
@ -184,7 +183,6 @@ class TestHTTPFlow:
def test_resume(self):
f = tflow.tflow()
f.reply.handle()
f.intercept()
assert f.reply.state == "taken"
f.resume()

12
test/mitmproxy/test_taddons.py Normal file
View File

@ -0,0 +1,12 @@
from mitmproxy.test import taddons
from mitmproxy import ctx
def test_recordingmaster():
with taddons.context() as tctx:
assert not tctx.master.has_log("nonexistent")
assert not tctx.master.has_event("nonexistent")
ctx.log.error("foo")
assert not tctx.master.has_log("foo", level="debug")
assert tctx.master.has_log("foo", level="error")

6
test/mitmproxy/tools/console/test_master.py
View File

@ -52,11 +52,11 @@ class TestMaster(tservers.MasterTest):
"""regression test for https://github.com/mitmproxy/mitmproxy/issues/1605"""
m = self.mkmaster(intercept="~b bar")
f = tflow.tflow(req=tutils.treq(content=b"foo"))
m.request(f)
m.addons.handle_lifecycle("request", f)
assert not m.view[0].intercepted
f = tflow.tflow(req=tutils.treq(content=b"bar"))
m.request(f)
m.addons.handle_lifecycle("request", f)
assert m.view[1].intercepted
f = tflow.tflow(resp=tutils.tresp(content=b"bar"))
m.request(f)
m.addons.handle_lifecycle("request", f)
assert m.view[2].intercepted

2
test/mitmproxy/tools/test_dump.py
View File

@ -20,7 +20,7 @@ class TestDumpMaster(tservers.MasterTest):
m = self.mkmaster(None)
ent = log.LogEntry("foo", "error")
ent.reply = controller.DummyReply()
m.log(ent)
m.addons.trigger("log", ent)
assert m.errorcheck.has_errored
@pytest.mark.parametrize("termlog", [False, True])

3
test/mitmproxy/tools/web/test_app.py
View File

@ -83,7 +83,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
def test_resume(self):
for f in self.view:
f.reply.handle()
f.intercept()
assert self.fetch(
@ -95,7 +94,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
def test_kill(self):
for f in self.view:
f.backup()
f.reply.handle()
f.intercept()
assert self.fetch("/flows/42/kill", method="POST").code == 200
@ -109,7 +107,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
f = self.view.get_by_id("42")
assert f
f.reply.handle()
assert self.fetch("/flows/42", method="DELETE").code == 200
assert not self.view.get_by_id("42")

15
test/mitmproxy/tservers.py
View File

@ -10,10 +10,10 @@ from mitmproxy import controller
from mitmproxy import options
from mitmproxy import exceptions
from mitmproxy import io
from mitmproxy import http
import pathod.test
import pathod.pathoc
from mitmproxy import eventsequence
from mitmproxy.test import tflow
from mitmproxy.test import tutils
from mitmproxy.test import taddons
@ -23,15 +23,10 @@ class MasterTest:
def cycle(self, master, content):
f = tflow.tflow(req=tutils.treq(content=content))
master.clientconnect(f.client_conn)
master.serverconnect(f.server_conn)
master.request(f)
if not f.error:
f.response = http.HTTPResponse.wrap(
tutils.tresp(content=content)
)
master.response(f)
master.clientdisconnect(f)
master.addons.handle_lifecycle("clientconnect", f.client_conn)
for i in eventsequence.iterate(f):
master.addons.handle_lifecycle(*i)
master.addons.handle_lifecycle("clientdisconnect", f.client_conn)
return f
def dummy_cycle(self, master, n, content):