improve HTTPRequest syntax

This commit is contained in:
Maximilian Hils 2014-09-03 23:44:54 +02:00
parent cd43c5ba9c
commit 2f44b26b4c
14 changed files with 152 additions and 188 deletions

View File

@ -1,8 +1,8 @@
def request(context, flow):
if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
frm = flow.request.get_form_urlencoded()
frm = flow.request.form_urlencoded
frm["mitmproxy"] = ["rocks"]
flow.request.set_form_urlencoded(frm)
flow.request.form_urlencoded = frm

View File

@ -1,7 +1,7 @@
def request(context, flow):
q = flow.request.get_query()
q = flow.request.query
if q:
q["mitmproxy"] = ["rocks"]
flow.request.set_query(q)
flow.request.query = q

View File

@ -7,12 +7,12 @@ This example shows two ways to redirect flows to other destinations.
def request(context, flow):
if flow.request.get_host(hostheader=True).endswith("example.com"):
if flow.request.pretty_host(hostheader=True).endswith("example.com"):
resp = HTTPResponse(
[1, 1], 200, "OK",
ODictCaseless([["Content-Type", "text/html"]]),
"helloworld")
flow.request.reply(resp)
if flow.request.get_host(hostheader=True).endswith("example.org"):
if flow.request.pretty_host(hostheader=True).endswith("example.org"):
flow.request.host = "mitmproxy.org"
flow.request.headers["Host"] = ["mitmproxy.org"]
flow.request.update_host_header()

View File

@ -177,7 +177,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
req_is_replay = f.request.is_replay,
req_method = f.request.method,
req_acked = f.request.reply.acked,
req_url = f.request.get_url(hostheader=hostheader),
req_url = f.request.pretty_url(hostheader=hostheader),
err_msg = f.error.msg if f.error else None,
resp_code = f.response.code if f.response else None,

View File

@ -528,7 +528,9 @@ class FlowView(common.WWrap):
def set_url(self, url):
request = self.flow.request
if not request.set_url(str(url)):
try:
request.url = str(url)
except ValueError:
return "Invalid URL."
self.master.refresh_flow(self.flow)
@ -552,17 +554,17 @@ class FlowView(common.WWrap):
conn.headers = flow.ODictCaseless(lst)
def set_query(self, lst, conn):
conn.set_query(flow.ODict(lst))
conn.query = flow.ODict(lst)
def set_path_components(self, lst, conn):
conn.set_path_components([i[0] for i in lst])
conn.path_components = [i[0] for i in lst]
def set_form(self, lst, conn):
conn.set_form_urlencoded(flow.ODict(lst))
conn.form_urlencoded = flow.ODict(lst)
def edit_form(self, conn):
self.master.view_grideditor(
grideditor.URLEncodedFormEditor(self.master, conn.get_form_urlencoded().lst, self.set_form, conn)
grideditor.URLEncodedFormEditor(self.master, conn.form_urlencoded.lst, self.set_form, conn)
)
def edit_form_confirm(self, key, conn):
@ -587,7 +589,7 @@ class FlowView(common.WWrap):
c = self.master.spawn_editor(conn.content or "")
conn.content = c.rstrip("\n") # what?
elif part == "f":
if not conn.get_form_urlencoded() and conn.content:
if not conn.form_urlencoded and conn.content:
self.master.prompt_onekey(
"Existing body is not a URL-encoded form. Clear and edit?",
[
@ -602,13 +604,13 @@ class FlowView(common.WWrap):
elif part == "h":
self.master.view_grideditor(grideditor.HeaderEditor(self.master, conn.headers.lst, self.set_headers, conn))
elif part == "p":
p = conn.get_path_components()
p = conn.path_components
p = [[i] for i in p]
self.master.view_grideditor(grideditor.PathEditor(self.master, p, self.set_path_components, conn))
elif part == "q":
self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn))
self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.query.lst, self.set_query, conn))
elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
self.master.prompt_edit("URL", conn.get_url(), self.set_url)
self.master.prompt_edit("URL", conn.url, self.set_url)
elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST:
self.master.prompt_onekey("Method", self.method_options, self.edit_method)
elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE:

View File

@ -55,7 +55,7 @@ def str_request(f, showhost):
c = f.client_conn.address.host
else:
c = "[replay]"
r = "%s %s %s"%(c, f.request.method, f.request.get_url(showhost, f))
r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
if f.request.stickycookie:
r = "[stickycookie] " + r
return r

View File

@ -208,7 +208,7 @@ class FDomain(_Rex):
code = "d"
help = "Domain"
def __call__(self, f):
return bool(re.search(self.expr, f.request.get_host(False, f), re.IGNORECASE))
return bool(re.search(self.expr, f.request.host, re.IGNORECASE))
class FUrl(_Rex):
@ -222,7 +222,7 @@ class FUrl(_Rex):
return klass(*toks)
def __call__(self, f):
return re.search(self.expr, f.request.get_url(False, f))
return re.search(self.expr, f.request.url)
class _Int(_Action):

View File

@ -260,8 +260,8 @@ class StickyCookieState:
Returns a (domain, port, path) tuple.
"""
return (
m["domain"] or f.request.get_host(False, f),
f.request.get_port(f),
m["domain"] or f.request.host,
f.request.port,
m["path"] or "/"
)
@ -279,7 +279,7 @@ class StickyCookieState:
c = Cookie.SimpleCookie(str(i))
m = c.values()[0]
k = self.ckey(m, f)
if self.domain_match(f.request.get_host(False, f), k[0]):
if self.domain_match(f.request.host, k[0]):
self.jar[self.ckey(m, f)] = m
def handle_request(self, f):
@ -287,8 +287,8 @@ class StickyCookieState:
if f.match(self.flt):
for i in self.jar.keys():
match = [
self.domain_match(f.request.get_host(False, f), i[0]),
f.request.get_port(f) == i[1],
self.domain_match(f.request.host, i[0]),
f.request.port == i[1],
f.request.path.startswith(i[2])
]
if all(match):
@ -307,7 +307,7 @@ class StickyAuthState:
self.hosts = {}
def handle_request(self, f):
host = f.request.get_host(False, f)
host = f.request.host
if "authorization" in f.request.headers:
self.hosts[host] = f.request.headers["authorization"]
elif f.match(self.flt):

View File

@ -411,7 +411,14 @@ class HTTPRequest(HTTPMessage):
e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0]
)]
def get_form_urlencoded(self):
def update_host_header(self):
"""
Update the host header to reflect the current target.
"""
self.headers["Host"] = [self.host]
@property
def form_urlencoded(self):
"""
Retrieves the URL-encoded form data, returning an ODict object.
Returns an empty ODict if there is no data or the content-type
@ -421,7 +428,8 @@ class HTTPRequest(HTTPMessage):
return ODict(utils.urldecode(self.content))
return ODict([])
def set_form_urlencoded(self, odict):
@form_urlencoded.setter
def form_urlencoded(self, odict):
"""
Sets the body to the URL-encoded form data, and adds the
appropriate content-type header. Note that this will destory the
@ -432,16 +440,18 @@ class HTTPRequest(HTTPMessage):
self.headers["Content-Type"] = [HDR_FORM_URLENCODED]
self.content = utils.urlencode(odict.lst)
def get_path_components(self, f):
@property
def path_components(self):
"""
Returns the path components of the URL as a list of strings.
Components are unquoted.
"""
_, _, path, _, _, _ = urlparse.urlparse(self.get_url(False, f))
_, _, path, _, _, _ = urlparse.urlparse(self.url)
return [urllib.unquote(i) for i in path.split("/") if i]
def set_path_components(self, lst, f):
@path_components.setter
def path_components(self, lst):
"""
Takes a list of strings, and sets the path component of the URL.
@ -449,32 +459,34 @@ class HTTPRequest(HTTPMessage):
"""
lst = [urllib.quote(i, safe="") for i in lst]
path = "/" + "/".join(lst)
scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.get_url(False, f))
self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f)
scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.url)
self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment])
def get_query(self, f):
@property
def query(self):
"""
Gets the request query string. Returns an ODict object.
"""
_, _, _, _, query, _ = urlparse.urlparse(self.get_url(False, f))
_, _, _, _, query, _ = urlparse.urlparse(self.url)
if query:
return ODict(utils.urldecode(query))
return ODict([])
def set_query(self, odict, f):
@query.setter
def query(self, odict):
"""
Takes an ODict object, and sets the request query string.
"""
scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.get_url(False, f))
scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.url)
query = utils.urlencode(odict.lst)
self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f)
self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment])
def get_host(self, hostheader, flow):
def pretty_host(self, hostheader):
"""
Heuristic to get the host of the request.
Note that get_host() does not always return the TCP destination of the request,
e.g. on a transparently intercepted request to an unrelated HTTP proxy.
Note that pretty_host() does not always return the TCP destination of the request,
e.g. if an upstream proxy is in place
If hostheader is set to True, the Host: header will be used as additional (and preferred) data source.
This is handy in transparent mode, where only the ip of the destination is known, but not the
@ -484,54 +496,27 @@ class HTTPRequest(HTTPMessage):
if hostheader:
host = self.headers.get_first("host")
if not host:
if self.host:
host = self.host
else:
for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect":
host = s[1]["host"]
break
if not host:
host = flow.server_conn.address.host
host = self.host
host = host.encode("idna")
return host
def get_scheme(self, flow):
"""
Returns the request port, either from the request itself or from the flow's server connection
"""
if self.scheme:
return self.scheme
if self.form_out == "authority": # On SSLed connections, the original CONNECT request is still unencrypted.
return "http"
return "https" if flow.server_conn.ssl_established else "http"
def get_port(self, flow):
"""
Returns the request port, either from the request itself or from the flow's server connection
"""
if self.port:
return self.port
for s in flow.server_conn.state:
if s[0] == "http" and s[1].get("state") == "connect":
return s[1]["port"]
return flow.server_conn.address.port
def get_url(self, hostheader, flow):
"""
Returns a URL string, constructed from the Request's URL components.
If hostheader is True, we use the value specified in the request
Host header to construct the URL.
"""
def pretty_url(self, hostheader):
if self.form_out == "authority": # upstream proxy mode
return "%s:%s" % (self.get_host(hostheader, flow), self.get_port(flow))
return utils.unparse_url(self.get_scheme(flow),
self.get_host(hostheader, flow),
self.get_port(flow),
return "%s:%s" % (self.pretty_host(hostheader), self.port)
return utils.unparse_url(self.scheme,
self.pretty_host(hostheader),
self.port,
self.path).encode('ascii')
def set_url(self, url, flow):
@property
def url(self):
"""
Returns a URL string, constructed from the Request's URL components.
"""
return self.pretty_url(False)
@url.setter
def url(self, url):
"""
Parses a URL specification, and updates the Request's information
accordingly.
@ -540,31 +525,11 @@ class HTTPRequest(HTTPMessage):
"""
parts = http.parse_url(url)
if not parts:
return False
scheme, host, port, path = parts
is_ssl = (True if scheme == "https" else False)
raise ValueError("Invalid URL: %s" % url)
self.scheme, self.host, self.port, self.path = parts
self.path = path
if host != self.get_host(False, flow) or port != self.get_port(flow):
if flow.live:
flow.live.change_server((host, port), ssl=is_ssl)
else:
# There's not live server connection, we're just changing the attributes here.
flow.server_conn = ServerConnection((host, port))
flow.server_conn.ssl_established = is_ssl
# If this is an absolute request, replace the attributes on the request object as well.
if self.host:
self.host = host
if self.port:
self.port = port
if self.scheme:
self.scheme = scheme
return True
def get_cookies(self):
@property
def cookies(self):
cookie_headers = self.headers.get("cookie")
if not cookie_headers:
return None
@ -760,7 +725,8 @@ class HTTPResponse(HTTPMessage):
if c:
self.headers["set-cookie"] = c
def get_cookies(self):
@property
def cookies(self):
cookie_headers = self.headers.get("set-cookie")
if not cookie_headers:
return None
@ -1127,12 +1093,12 @@ class HTTPHandler(ProtocolHandler):
if not request.host:
# Host/Port Complication: In upstream mode, use the server we CONNECTed to,
# not the upstream proxy.
for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect":
request.host, request.port = s[1]["host"], s[1]["port"]
if not request.host:
request.host = flow.server_conn.address.host
request.port = flow.server_conn.address.port
if flow.server_conn:
for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect":
request.host, request.port = s[1]["host"], s[1]["port"]
if not request.host and flow.server_conn:
request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port
# Now we can process the request.
if request.form_in == "authority":
@ -1242,7 +1208,9 @@ class RequestReplayThread(threading.Thread):
r.form_out = self.config.http_form_out
server_address, server_ssl = False, False
if self.config.get_upstream_server:
# If the flow is live, r.host is already the correct upstream server unless modified by a script.
# If modified by a script, we probably want to keep the modified destination.
if self.config.get_upstream_server and not self.flow.live:
try:
# this will fail in transparent mode
upstream_info = self.config.get_upstream_server(self.flow.client_conn)
@ -1251,17 +1219,16 @@ class RequestReplayThread(threading.Thread):
except proxy.ProxyError:
pass
if not server_address:
server_address = (r.get_host(False, self.flow), r.get_port(self.flow))
server_address = (r.host, r.port)
server = ServerConnection(server_address, None)
server = ServerConnection(server_address)
server.connect()
if server_ssl or r.get_scheme(self.flow) == "https":
if server_ssl or r.scheme == "https":
if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode -> send CONNECT
send_connect_request(server, r.get_host(), r.get_port())
send_connect_request(server, r.host, r.port)
r.form_out = "relative"
server.establish_ssl(self.config.clientcerts,
self.flow.server_conn.sni)
server.establish_ssl(self.config.clientcerts, sni=r.host)
server.send(r._assemble())
self.flow.response = HTTPResponse.from_stream(server.rfile, r.method,
body_size_limit=self.config.body_size_limit)

View File

@ -58,7 +58,7 @@ class Flow(stateobject.SimpleStateObject):
"""@type: ClientConnection"""
self.server_conn = server_conn
"""@type: ServerConnection"""
self.live = live # Used by flow.request.set_url to change the server address
self.live = live
"""@type: LiveConnection"""
self.error = None

View File

@ -753,10 +753,10 @@ class TestRequest:
def test_simple(self):
f = tutils.tflow()
r = f.request
u = r.get_url(False, f)
assert r.set_url(u, f)
assert not r.set_url("", f)
assert r.get_url(False, f) == u
u = r.url
r.url = u
tutils.raises(ValueError, setattr, r, "url", "")
assert r.url == u
assert r._assemble()
assert r.size() == len(r._assemble())
@ -771,83 +771,81 @@ class TestRequest:
tutils.raises("Cannot assemble flow with CONTENT_MISSING", r._assemble)
def test_get_url(self):
f = tutils.tflow()
r = f.request
r = tutils.treq()
assert r.get_url(False, f) == "http://address:22/path"
assert r.url == "http://address:22/path"
r.scheme = "https"
assert r.get_url(False, f) == "https://address:22/path"
assert r.url == "https://address:22/path"
r.host = "host"
r.port = 42
assert r.get_url(False, f) == "https://host:42/path"
assert r.url == "https://host:42/path"
r.host = "address"
r.port = 22
assert r.get_url(False, f) == "https://address:22/path"
assert r.url== "https://address:22/path"
assert r.get_url(True, f) == "https://address:22/path"
assert r.pretty_url(True) == "https://address:22/path"
r.headers["Host"] = ["foo.com"]
assert r.get_url(False, f) == "https://address:22/path"
assert r.get_url(True, f) == "https://foo.com:22/path"
assert r.pretty_url(False) == "https://address:22/path"
assert r.pretty_url(True) == "https://foo.com:22/path"
def test_path_components(self):
f = tutils.tflow()
r = f.request
r = tutils.treq()
r.path = "/"
assert r.get_path_components(f) == []
assert r.path_components == []
r.path = "/foo/bar"
assert r.get_path_components(f) == ["foo", "bar"]
assert r.path_components == ["foo", "bar"]
q = flow.ODict()
q["test"] = ["123"]
r.set_query(q, f)
assert r.get_path_components(f) == ["foo", "bar"]
r.query = q
assert r.path_components == ["foo", "bar"]
r.set_path_components([], f)
assert r.get_path_components(f) == []
r.set_path_components(["foo"], f)
assert r.get_path_components(f) == ["foo"]
r.set_path_components(["/oo"], f)
assert r.get_path_components(f) == ["/oo"]
r.path_components = []
assert r.path_components == []
r.path_components = ["foo"]
assert r.path_components == ["foo"]
r.path_components = ["/oo"]
assert r.path_components == ["/oo"]
assert "%2F" in r.path
def test_getset_form_urlencoded(self):
d = flow.ODict([("one", "two"), ("three", "four")])
r = tutils.treq(content=utils.urlencode(d.lst))
r.headers["content-type"] = [protocol.http.HDR_FORM_URLENCODED]
assert r.get_form_urlencoded() == d
assert r.form_urlencoded == d
d = flow.ODict([("x", "y")])
r.set_form_urlencoded(d)
assert r.get_form_urlencoded() == d
r.form_urlencoded = d
assert r.form_urlencoded == d
r.headers["content-type"] = ["foo"]
assert not r.get_form_urlencoded()
assert not r.form_urlencoded
def test_getset_query(self):
h = flow.ODictCaseless()
f = tutils.tflow()
f.request.path = "/foo?x=y&a=b"
q = f.request.get_query(f)
r = tutils.treq()
r.path = "/foo?x=y&a=b"
q = r.query
assert q.lst == [("x", "y"), ("a", "b")]
f.request.path = "/"
q = f.request.get_query(f)
r.path = "/"
q = r.query
assert not q
f.request.path = "/?adsfa"
q = f.request.get_query(f)
r.path = "/?adsfa"
q = r.query
assert q.lst == [("adsfa", "")]
f.request.path = "/foo?x=y&a=b"
assert f.request.get_query(f)
f.request.set_query(flow.ODict([]), f)
assert not f.request.get_query(f)
r.path = "/foo?x=y&a=b"
assert r.query
r.query = flow.ODict([])
assert not r.query
qv = flow.ODict([("a", "b"), ("c", "d")])
f.request.set_query(qv, f)
assert f.request.get_query(f) == qv
r.query = qv
assert r.query == qv
def test_anticache(self):
h = flow.ODictCaseless()
@ -918,14 +916,14 @@ class TestRequest:
h = flow.ODictCaseless()
r = tutils.treq()
r.headers = h
assert r.get_cookies() is None
assert r.cookies is None
def test_get_cookies_single(self):
h = flow.ODictCaseless()
h["Cookie"] = ["cookiename=cookievalue"]
r = tutils.treq()
r.headers = h
result = r.get_cookies()
result = r.cookies
assert len(result)==1
assert result['cookiename']==('cookievalue',{})
@ -934,7 +932,7 @@ class TestRequest:
h["Cookie"] = ["cookiename=cookievalue;othercookiename=othercookievalue"]
r = tutils.treq()
r.headers = h
result = r.get_cookies()
result = r.cookies
assert len(result)==2
assert result['cookiename']==('cookievalue',{})
assert result['othercookiename']==('othercookievalue',{})
@ -944,7 +942,7 @@ class TestRequest:
h["Cookie"] = ["cookiename=coo=kievalue;othercookiename=othercookievalue"]
r = tutils.treq()
r.headers = h
result = r.get_cookies()
result = r.cookies
assert len(result)==2
assert result['cookiename']==('coo=kievalue',{})
assert result['othercookiename']==('othercookievalue',{})
@ -1054,14 +1052,14 @@ class TestResponse:
h = flow.ODictCaseless()
resp = tutils.tresp()
resp.headers = h
assert not resp.get_cookies()
assert not resp.cookies
def test_get_cookies_simple(self):
h = flow.ODictCaseless()
h["Set-Cookie"] = ["cookiename=cookievalue"]
resp = tutils.tresp()
resp.headers = h
result = resp.get_cookies()
result = resp.cookies
assert len(result)==1
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", {})
@ -1071,7 +1069,7 @@ class TestResponse:
h["Set-Cookie"] = ["cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly"]
resp = tutils.tresp()
resp.headers = h
result = resp.get_cookies()
result = resp.cookies
assert len(result)==1
assert "cookiename" in result
assert result["cookiename"][0] == "cookievalue"
@ -1086,7 +1084,7 @@ class TestResponse:
h["Set-Cookie"] = ["cookiename=; Expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/"]
resp = tutils.tresp()
resp.headers = h
result = resp.get_cookies()
result = resp.cookies
assert len(result)==1
assert "cookiename" in result
assert result["cookiename"][0] == ""
@ -1097,7 +1095,7 @@ class TestResponse:
h["Set-Cookie"] = ["cookiename=cookievalue","othercookie=othervalue"]
resp = tutils.tresp()
resp.headers = h
result = resp.get_cookies()
result = resp.cookies
assert len(result)==2
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", {})

View File

@ -58,12 +58,12 @@ class TestHTTPRequest:
tutils.raises("Invalid request form", r._assemble, "antiauthority")
def test_set_url(self):
f = tutils.tflow(req=tutils.treq_absolute())
f.request.set_url("https://otheraddress:42/ORLY", f)
assert f.request.scheme == "https"
assert f.request.host == "otheraddress"
assert f.request.port == 42
assert f.request.path == "/ORLY"
r = tutils.treq_absolute()
r.url = "https://otheraddress:42/ORLY"
assert r.scheme == "https"
assert r.host == "otheraddress"
assert r.port == 42
assert r.path == "/ORLY"
class TestHTTPResponse:

View File

@ -23,7 +23,7 @@ class TestServerConnection:
self.d.shutdown()
def test_simple(self):
sc = ServerConnection((self.d.IFACE, self.d.port), None)
sc = ServerConnection((self.d.IFACE, self.d.port))
sc.connect()
f = tutils.tflow()
f.server_conn = sc
@ -35,7 +35,7 @@ class TestServerConnection:
sc.finish()
def test_terminate_error(self):
sc = ServerConnection((self.d.IFACE, self.d.port), None)
sc = ServerConnection((self.d.IFACE, self.d.port))
sc.connect()
sc.connection = mock.Mock()
sc.connection.recv = mock.Mock(return_value=False)

View File

@ -337,17 +337,14 @@ class MasterRedirectRequest(tservers.TestMaster):
def handle_request(self, f):
request = f.request
if request.path == "/p/201":
url = request.get_url(False, f)
url = request.url
new = "http://127.0.0.1:%s/p/201" % self.redirect_port
request.set_url(new, f)
request.set_url(new, f)
request.url = new
f.live.change_server(("127.0.0.1", self.redirect_port), False)
request.set_url(url, f)
request.url = url
tutils.raises("SSL handshake error", f.live.change_server, ("127.0.0.1", self.redirect_port), True)
request.set_url(new, f)
request.set_url(url, f)
request.set_url(new, f)
request.url = new
tservers.TestMaster.handle_request(self, f)
def handle_response(self, f):