mirror of
https://github.com/Grasscutters/mitmproxy.git
synced 2024-11-23 00:01:36 +00:00
remove old code
This commit is contained in:
Maximilian Hils
parent
08655cb956
commit
3873e08339
@ -1,37 +0,0 @@
|
|||||||
# This script makes mitmproxy switch to passthrough mode for all HTTP
|
|
||||||
# responses with "Connection: Upgrade" header. This is useful to make
|
|
||||||
# WebSockets work in untrusted environments.
|
|
||||||
#
|
|
||||||
# Note: Chrome (and possibly other browsers), when explicitly configured
|
|
||||||
# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request
|
|
||||||
# to the proxy before they initiate the websocket connection.
|
|
||||||
# To make WebSockets work in these cases, supply
|
|
||||||
# `--ignore :80$` as an additional parameter.
|
|
||||||
# (see http://mitmproxy.org/doc/features/passthrough.html)
|
|
||||||
|
|
||||||
import netlib.http.semantics
|
|
||||||
|
|
||||||
from libmproxy.protocol.tcp import TCPHandler
|
|
||||||
from libmproxy.protocol import KILL
|
|
||||||
from libmproxy.script import concurrent
|
|
||||||
|
|
||||||
|
|
||||||
def start(context, argv):
|
|
||||||
netlib.http.semantics.Request._headers_to_strip_off.remove("Connection")
|
|
||||||
netlib.http.semantics.Request._headers_to_strip_off.remove("Upgrade")
|
|
||||||
|
|
||||||
|
|
||||||
def done(context):
|
|
||||||
netlib.http.semantics.Request._headers_to_strip_off.append("Connection")
|
|
||||||
netlib.http.semantics.Request._headers_to_strip_off.append("Upgrade")
|
|
||||||
|
|
||||||
|
|
||||||
@concurrent
|
|
||||||
def response(context, flow):
|
|
||||||
value = flow.response.headers.get_first("Connection", None)
|
|
||||||
if value and value.upper() == "UPGRADE":
|
|
||||||
# We need to send the response manually now...
|
|
||||||
flow.client_conn.send(flow.client_conn.protocol.assemble(flow.response))
|
|
||||||
# ...and then delegate to tcp passthrough.
|
|
||||||
TCPHandler(flow.live.c, log=False).handle_messages()
|
|
||||||
flow.reply(KILL)
|
|
||||||
@ -8,15 +8,16 @@ import Cookie
|
|||||||
import cookielib
|
import cookielib
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
from libmproxy.protocol.http import HTTPFlow
|
||||||
from libmproxy.protocol2.http_replay import RequestReplayThread
|
from libmproxy.protocol2.http_replay import RequestReplayThread
|
||||||
|
|
||||||
from netlib import odict, wsgi, tcp
|
from netlib import odict, wsgi
|
||||||
from netlib.http.semantics import CONTENT_MISSING
|
from netlib.http.semantics import CONTENT_MISSING
|
||||||
import netlib.http
|
import netlib.http
|
||||||
|
|
||||||
from . import controller, protocol, tnetstring, filt, script, version
|
from . import controller, protocol, tnetstring, filt, script, version
|
||||||
from .onboarding import app
|
from .onboarding import app
|
||||||
from .protocol import http, handle
|
from .protocol import http
|
||||||
from .proxy.config import HostMatcher
|
from .proxy.config import HostMatcher
|
||||||
from .proxy.connection import ClientConnection, ServerConnection
|
from .proxy.connection import ClientConnection, ServerConnection
|
||||||
import urlparse
|
import urlparse
|
||||||
@ -1090,7 +1091,7 @@ class FlowReader:
|
|||||||
"Incompatible serialized data version: %s" % v
|
"Incompatible serialized data version: %s" % v
|
||||||
)
|
)
|
||||||
off = self.fo.tell()
|
off = self.fo.tell()
|
||||||
yield handle.protocols[data["type"]]["flow"].from_state(data)
|
yield HTTPFlow.from_state(data)
|
||||||
except ValueError as v:
|
except ValueError as v:
|
||||||
# Error is due to EOF
|
# Error is due to EOF
|
||||||
if self.fo.tell() == off and self.fo.read() == '':
|
if self.fo.tell() == off and self.fo.read() == '':
|
||||||
|
|||||||
@ -1,20 +0,0 @@
|
|||||||
from __future__ import absolute_import
|
|
||||||
from . import http, tcp
|
|
||||||
|
|
||||||
protocols = {
|
|
||||||
'http': dict(handler=http.HTTPHandler, flow=http.HTTPFlow),
|
|
||||||
'tcp': dict(handler=tcp.TCPHandler)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def protocol_handler(protocol):
|
|
||||||
"""
|
|
||||||
@type protocol: str
|
|
||||||
@returns: libmproxy.protocol.primitives.ProtocolHandler
|
|
||||||
"""
|
|
||||||
if protocol in protocols:
|
|
||||||
return protocols[protocol]["handler"]
|
|
||||||
|
|
||||||
raise NotImplementedError(
|
|
||||||
"Unknown Protocol: %s" %
|
|
||||||
protocol) # pragma: nocover
|
|
||||||
@ -1,62 +1,9 @@
|
|||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
import Cookie
|
|
||||||
import copy
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
import urllib
|
|
||||||
import urlparse
|
|
||||||
from email.utils import parsedate_tz, formatdate, mktime_tz
|
|
||||||
|
|
||||||
import netlib
|
from .primitives import Flow
|
||||||
from netlib import http, tcp, odict, utils, encoding
|
|
||||||
from netlib.http import cookies, http1, http2
|
|
||||||
from netlib.http.http1 import HTTP1Protocol
|
|
||||||
from netlib.http.semantics import CONTENT_MISSING
|
|
||||||
|
|
||||||
from .tcp import TCPHandler
|
|
||||||
from .primitives import KILL, ProtocolHandler, Flow, Error
|
|
||||||
from ..proxy.connection import ServerConnection
|
|
||||||
from .. import utils, controller, stateobject, proxy
|
|
||||||
|
|
||||||
from .http_wrappers import decoded, HTTPRequest, HTTPResponse
|
from .http_wrappers import decoded, HTTPRequest, HTTPResponse
|
||||||
|
|
||||||
|
|
||||||
class KillSignal(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def send_connect_request(conn, host, port, update_state=True):
|
|
||||||
upstream_request = HTTPRequest(
|
|
||||||
"authority",
|
|
||||||
"CONNECT",
|
|
||||||
None,
|
|
||||||
host,
|
|
||||||
port,
|
|
||||||
None,
|
|
||||||
(1, 1),
|
|
||||||
odict.ODictCaseless(),
|
|
||||||
""
|
|
||||||
)
|
|
||||||
|
|
||||||
# we currently only support HTTP/1 CONNECT requests
|
|
||||||
protocol = http1.HTTP1Protocol(conn)
|
|
||||||
|
|
||||||
conn.send(protocol.assemble(upstream_request))
|
|
||||||
resp = HTTPResponse.from_protocol(protocol, upstream_request.method)
|
|
||||||
if resp.status_code != 200:
|
|
||||||
raise proxy.ProxyError(resp.status_code,
|
|
||||||
"Cannot establish SSL " +
|
|
||||||
"connection with upstream proxy: \r\n" +
|
|
||||||
repr(resp))
|
|
||||||
if update_state:
|
|
||||||
conn.state.append(("http", {
|
|
||||||
"state": "connect",
|
|
||||||
"host": host,
|
|
||||||
"port": port}
|
|
||||||
))
|
|
||||||
return resp
|
|
||||||
|
|
||||||
|
|
||||||
class HTTPFlow(Flow):
|
class HTTPFlow(Flow):
|
||||||
"""
|
"""
|
||||||
A HTTPFlow is a collection of objects representing a single HTTP
|
A HTTPFlow is a collection of objects representing a single HTTP
|
||||||
@ -143,556 +90,3 @@ class HTTPFlow(Flow):
|
|||||||
if self.response:
|
if self.response:
|
||||||
c += self.response.replace(pattern, repl, *args, **kwargs)
|
c += self.response.replace(pattern, repl, *args, **kwargs)
|
||||||
return c
|
return c
|
||||||
|
|
||||||
|
|
||||||
class HTTPHandler(ProtocolHandler):
|
|
||||||
"""
|
|
||||||
HTTPHandler implements mitmproxys understanding of the HTTP protocol.
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, c):
|
|
||||||
super(HTTPHandler, self).__init__(c)
|
|
||||||
self.expected_form_in = c.config.mode.http_form_in
|
|
||||||
self.expected_form_out = c.config.mode.http_form_out
|
|
||||||
self.skip_authentication = False
|
|
||||||
|
|
||||||
def handle_messages(self):
|
|
||||||
while self.handle_flow():
|
|
||||||
pass
|
|
||||||
|
|
||||||
def get_response_from_server(self, flow):
|
|
||||||
self.c.establish_server_connection()
|
|
||||||
|
|
||||||
for attempt in (0, 1):
|
|
||||||
try:
|
|
||||||
if not self.c.server_conn.protocol:
|
|
||||||
# instantiate new protocol if connection does not have one yet
|
|
||||||
# TODO: select correct protocol based on ALPN (?)
|
|
||||||
self.c.server_conn.protocol = http1.HTTP1Protocol(self.c.server_conn)
|
|
||||||
# self.c.server_conn.protocol = http2.HTTP2Protocol(self.c.server_conn)
|
|
||||||
# self.c.server_conn.protocol.perform_connection_preface()
|
|
||||||
|
|
||||||
self.c.server_conn.send(self.c.server_conn.protocol.assemble(flow.request))
|
|
||||||
|
|
||||||
# Only get the headers at first...
|
|
||||||
flow.response = HTTPResponse.from_protocol(
|
|
||||||
self.c.server_conn.protocol,
|
|
||||||
flow.request.method,
|
|
||||||
body_size_limit=self.c.config.body_size_limit,
|
|
||||||
include_body=False,
|
|
||||||
)
|
|
||||||
break
|
|
||||||
except (tcp.NetLibError, http.HttpErrorConnClosed) as v:
|
|
||||||
self.c.log(
|
|
||||||
"error in server communication: %s" % repr(v),
|
|
||||||
level="debug"
|
|
||||||
)
|
|
||||||
if attempt == 0:
|
|
||||||
# In any case, we try to reconnect at least once. This is
|
|
||||||
# necessary because it might be possible that we already
|
|
||||||
# initiated an upstream connection after clientconnect that
|
|
||||||
# has already been expired, e.g consider the following event
|
|
||||||
# log:
|
|
||||||
# > clientconnect (transparent mode destination known)
|
|
||||||
# > serverconnect
|
|
||||||
# > read n% of large request
|
|
||||||
# > server detects timeout, disconnects
|
|
||||||
# > read (100-n)% of large request
|
|
||||||
# > send large request upstream
|
|
||||||
self.c.server_reconnect()
|
|
||||||
else:
|
|
||||||
raise
|
|
||||||
|
|
||||||
# call the appropriate script hook - this is an opportunity for an
|
|
||||||
# inline script to set flow.stream = True
|
|
||||||
flow = self.c.channel.ask("responseheaders", flow)
|
|
||||||
if flow is None or flow == KILL:
|
|
||||||
raise KillSignal()
|
|
||||||
else:
|
|
||||||
# now get the rest of the request body, if body still needs to be
|
|
||||||
# read but not streaming this response
|
|
||||||
if flow.response.stream:
|
|
||||||
flow.response.content = CONTENT_MISSING
|
|
||||||
else:
|
|
||||||
if isinstance(self.c.server_conn.protocol, http1.HTTP1Protocol):
|
|
||||||
# streaming is only supported with HTTP/1 at the moment
|
|
||||||
flow.response.content = self.c.server_conn.protocol.read_http_body(
|
|
||||||
flow.response.headers,
|
|
||||||
self.c.config.body_size_limit,
|
|
||||||
flow.request.method,
|
|
||||||
flow.response.code,
|
|
||||||
False
|
|
||||||
)
|
|
||||||
flow.response.timestamp_end = utils.timestamp()
|
|
||||||
|
|
||||||
def handle_flow(self):
|
|
||||||
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
|
|
||||||
|
|
||||||
try:
|
|
||||||
try:
|
|
||||||
if not flow.client_conn.protocol:
|
|
||||||
# instantiate new protocol if connection does not have one yet
|
|
||||||
# the first request might be a CONNECT - which is currently only supported with HTTP/1
|
|
||||||
flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn)
|
|
||||||
|
|
||||||
req = HTTPRequest.from_protocol(
|
|
||||||
flow.client_conn.protocol,
|
|
||||||
body_size_limit=self.c.config.body_size_limit
|
|
||||||
)
|
|
||||||
except tcp.NetLibError:
|
|
||||||
# don't throw an error for disconnects that happen
|
|
||||||
# before/between requests.
|
|
||||||
return False
|
|
||||||
|
|
||||||
self.c.log(
|
|
||||||
"request",
|
|
||||||
"debug",
|
|
||||||
[repr(req)]
|
|
||||||
)
|
|
||||||
ret = self.process_request(flow, req)
|
|
||||||
if ret:
|
|
||||||
# instantiate new protocol if connection does not have one yet
|
|
||||||
# TODO: select correct protocol based on ALPN (?)
|
|
||||||
flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn)
|
|
||||||
# flow.client_conn.protocol = http2.HTTP2Protocol(self.c.client_conn, is_server=True)
|
|
||||||
if ret is not None:
|
|
||||||
return ret
|
|
||||||
|
|
||||||
# Be careful NOT to assign the request to the flow before
|
|
||||||
# process_request completes. This is because the call can raise an
|
|
||||||
# exception. If the request object is already attached, this results
|
|
||||||
# in an Error object that has an attached request that has not been
|
|
||||||
# sent through to the Master.
|
|
||||||
flow.request = req
|
|
||||||
request_reply = self.c.channel.ask("request", flow)
|
|
||||||
if request_reply is None or request_reply == KILL:
|
|
||||||
raise KillSignal()
|
|
||||||
|
|
||||||
# The inline script may have changed request.host
|
|
||||||
self.process_server_address(flow)
|
|
||||||
|
|
||||||
if isinstance(request_reply, HTTPResponse):
|
|
||||||
flow.response = request_reply
|
|
||||||
else:
|
|
||||||
self.get_response_from_server(flow)
|
|
||||||
|
|
||||||
# no further manipulation of self.c.server_conn beyond this point
|
|
||||||
# we can safely set it as the final attribute value here.
|
|
||||||
flow.server_conn = self.c.server_conn
|
|
||||||
|
|
||||||
self.c.log(
|
|
||||||
"response",
|
|
||||||
"debug",
|
|
||||||
[repr(flow.response)]
|
|
||||||
)
|
|
||||||
response_reply = self.c.channel.ask("response", flow)
|
|
||||||
if response_reply is None or response_reply == KILL:
|
|
||||||
raise KillSignal()
|
|
||||||
|
|
||||||
self.send_response_to_client(flow)
|
|
||||||
|
|
||||||
if self.check_close_connection(flow):
|
|
||||||
return False
|
|
||||||
|
|
||||||
# We sent a CONNECT request to an upstream proxy.
|
|
||||||
if flow.request.form_in == "authority" and flow.response.code == 200:
|
|
||||||
# TODO: Possibly add headers (memory consumption/usefulness
|
|
||||||
# tradeoff) Make sure to add state info before the actual
|
|
||||||
# processing of the CONNECT request happens. During an SSL
|
|
||||||
# upgrade, we may receive an SNI indication from the client,
|
|
||||||
# which resets the upstream connection. If this is the case, we
|
|
||||||
# must already re-issue the CONNECT request at this point.
|
|
||||||
self.c.server_conn.state.append(
|
|
||||||
(
|
|
||||||
"http", {
|
|
||||||
"state": "connect",
|
|
||||||
"host": flow.request.host,
|
|
||||||
"port": flow.request.port
|
|
||||||
}
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if not self.process_connect_request(
|
|
||||||
(flow.request.host, flow.request.port)):
|
|
||||||
return False
|
|
||||||
|
|
||||||
# If the user has changed the target server on this connection,
|
|
||||||
# restore the original target server
|
|
||||||
flow.live.restore_server()
|
|
||||||
|
|
||||||
return True # Next flow please.
|
|
||||||
except (
|
|
||||||
http.HttpAuthenticationError,
|
|
||||||
http.HttpError,
|
|
||||||
proxy.ProxyError,
|
|
||||||
tcp.NetLibError,
|
|
||||||
) as e:
|
|
||||||
self.handle_error(e, flow)
|
|
||||||
except KillSignal:
|
|
||||||
self.c.log("Connection killed", "info")
|
|
||||||
finally:
|
|
||||||
flow.live = None # Connection is not live anymore.
|
|
||||||
return False
|
|
||||||
|
|
||||||
def handle_server_reconnect(self, state):
|
|
||||||
if state["state"] == "connect":
|
|
||||||
send_connect_request(
|
|
||||||
self.c.server_conn,
|
|
||||||
state["host"],
|
|
||||||
state["port"],
|
|
||||||
update_state=False
|
|
||||||
)
|
|
||||||
else: # pragma: nocover
|
|
||||||
raise RuntimeError("Unknown State: %s" % state["state"])
|
|
||||||
|
|
||||||
def handle_error(self, error, flow=None):
|
|
||||||
message = repr(error)
|
|
||||||
message_debug = None
|
|
||||||
|
|
||||||
if isinstance(error, tcp.NetLibError):
|
|
||||||
message = None
|
|
||||||
message_debug = "TCP connection closed unexpectedly."
|
|
||||||
elif "tlsv1 alert unknown ca" in message:
|
|
||||||
message = "TLSv1 Alert Unknown CA: The client does not trust the proxy's certificate."
|
|
||||||
elif "handshake error" in message:
|
|
||||||
message_debug = message
|
|
||||||
message = "SSL handshake error: The client may not trust the proxy's certificate."
|
|
||||||
|
|
||||||
if message:
|
|
||||||
self.c.log(message, level="info")
|
|
||||||
if message_debug:
|
|
||||||
self.c.log(message_debug, level="debug")
|
|
||||||
|
|
||||||
if flow:
|
|
||||||
# TODO: no flows without request or with both request and response
|
|
||||||
# at the moment.
|
|
||||||
if flow.request and not flow.response:
|
|
||||||
flow.error = Error(message or message_debug)
|
|
||||||
self.c.channel.ask("error", flow)
|
|
||||||
try:
|
|
||||||
status_code = getattr(error, "code", 502)
|
|
||||||
headers = getattr(error, "headers", None)
|
|
||||||
|
|
||||||
html_message = message or ""
|
|
||||||
if message_debug:
|
|
||||||
html_message += "<pre>%s</pre>" % message_debug
|
|
||||||
self.send_error(status_code, html_message, headers)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def send_error(self, status_code, message, headers):
|
|
||||||
response = http.status_codes.RESPONSES.get(status_code, "Unknown")
|
|
||||||
body = """
|
|
||||||
<html>
|
|
||||||
<head>
|
|
||||||
<title>%d %s</title>
|
|
||||||
</head>
|
|
||||||
<body>%s</body>
|
|
||||||
</html>
|
|
||||||
""" % (status_code, response, message)
|
|
||||||
|
|
||||||
if not headers:
|
|
||||||
headers = odict.ODictCaseless()
|
|
||||||
assert isinstance(headers, odict.ODictCaseless)
|
|
||||||
|
|
||||||
headers["Server"] = [self.c.config.server_version]
|
|
||||||
headers["Connection"] = ["close"]
|
|
||||||
headers["Content-Length"] = [len(body)]
|
|
||||||
headers["Content-Type"] = ["text/html"]
|
|
||||||
|
|
||||||
resp = HTTPResponse(
|
|
||||||
(1, 1), # if HTTP/2 is used, this value is ignored anyway
|
|
||||||
status_code,
|
|
||||||
response,
|
|
||||||
headers,
|
|
||||||
body,
|
|
||||||
)
|
|
||||||
|
|
||||||
# if no protocol is assigned yet - just assume HTTP/1
|
|
||||||
# TODO: maybe check ALPN and use HTTP/2 if required?
|
|
||||||
protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn)
|
|
||||||
self.c.client_conn.send(protocol.assemble(resp))
|
|
||||||
|
|
||||||
def process_request(self, flow, request):
|
|
||||||
"""
|
|
||||||
@returns:
|
|
||||||
True, if the request should not be sent upstream
|
|
||||||
False, if the connection should be aborted
|
|
||||||
None, if the request should be sent upstream
|
|
||||||
(a status code != None should be returned directly by handle_flow)
|
|
||||||
"""
|
|
||||||
|
|
||||||
if not self.skip_authentication:
|
|
||||||
self.authenticate(request)
|
|
||||||
|
|
||||||
# Determine .scheme, .host and .port attributes
|
|
||||||
# For absolute-form requests, they are directly given in the request.
|
|
||||||
# For authority-form requests, we only need to determine the request scheme.
|
|
||||||
# For relative-form requests, we need to determine host and port as
|
|
||||||
# well.
|
|
||||||
if not request.scheme:
|
|
||||||
request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http"
|
|
||||||
if not request.host:
|
|
||||||
# Host/Port Complication: In upstream mode, use the server we CONNECTed to,
|
|
||||||
# not the upstream proxy.
|
|
||||||
if flow.server_conn:
|
|
||||||
for s in flow.server_conn.state:
|
|
||||||
if s[0] == "http" and s[1]["state"] == "connect":
|
|
||||||
request.host, request.port = s[1]["host"], s[1]["port"]
|
|
||||||
if not request.host and flow.server_conn:
|
|
||||||
request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port
|
|
||||||
|
|
||||||
|
|
||||||
# Now we can process the request.
|
|
||||||
if request.form_in == "authority":
|
|
||||||
if self.c.client_conn.ssl_established:
|
|
||||||
raise http.HttpError(
|
|
||||||
400,
|
|
||||||
"Must not CONNECT on already encrypted connection"
|
|
||||||
)
|
|
||||||
|
|
||||||
if self.c.config.mode == "regular":
|
|
||||||
self.c.set_server_address((request.host, request.port))
|
|
||||||
# Update server_conn attribute on the flow
|
|
||||||
flow.server_conn = self.c.server_conn
|
|
||||||
|
|
||||||
# since we currently only support HTTP/1 CONNECT requests
|
|
||||||
# the response must be HTTP/1 as well
|
|
||||||
self.c.client_conn.send(
|
|
||||||
('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) +
|
|
||||||
'Connection established\r\n' +
|
|
||||||
'Content-Length: 0\r\n' +
|
|
||||||
('Proxy-agent: %s\r\n' % self.c.config.server_version) +
|
|
||||||
'\r\n'
|
|
||||||
)
|
|
||||||
return self.process_connect_request(self.c.server_conn.address)
|
|
||||||
elif self.c.config.mode == "upstream":
|
|
||||||
return None
|
|
||||||
else:
|
|
||||||
# CONNECT should never occur if we don't expect absolute-form
|
|
||||||
# requests
|
|
||||||
pass
|
|
||||||
|
|
||||||
elif request.form_in == self.expected_form_in:
|
|
||||||
request.form_out = self.expected_form_out
|
|
||||||
if request.form_in == "absolute":
|
|
||||||
if request.scheme != "http":
|
|
||||||
raise http.HttpError(
|
|
||||||
400,
|
|
||||||
"Invalid request scheme: %s" % request.scheme
|
|
||||||
)
|
|
||||||
if self.c.config.mode == "regular":
|
|
||||||
# Update info so that an inline script sees the correct
|
|
||||||
# value at flow.server_conn
|
|
||||||
self.c.set_server_address((request.host, request.port))
|
|
||||||
flow.server_conn = self.c.server_conn
|
|
||||||
|
|
||||||
elif request.form_in == "relative":
|
|
||||||
if self.c.config.mode == "spoof":
|
|
||||||
# Host header
|
|
||||||
h = request.pretty_host(hostheader=True)
|
|
||||||
if h is None:
|
|
||||||
raise http.HttpError(
|
|
||||||
400,
|
|
||||||
"Invalid request: No host information"
|
|
||||||
)
|
|
||||||
p = netlib.utils.parse_url("http://" + h)
|
|
||||||
request.scheme = p[0]
|
|
||||||
request.host = p[1]
|
|
||||||
request.port = p[2]
|
|
||||||
self.c.set_server_address((request.host, request.port))
|
|
||||||
flow.server_conn = self.c.server_conn
|
|
||||||
|
|
||||||
if self.c.config.mode == "sslspoof":
|
|
||||||
# SNI is processed in server.py
|
|
||||||
if not (flow.server_conn and flow.server_conn.ssl_established):
|
|
||||||
raise http.HttpError(
|
|
||||||
400,
|
|
||||||
"Invalid request: No host information"
|
|
||||||
)
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
raise http.HttpError(
|
|
||||||
400, "Invalid HTTP request form (expected: %s, got: %s)" % (
|
|
||||||
self.expected_form_in, request.form_in
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def process_server_address(self, flow):
|
|
||||||
# Depending on the proxy mode, server handling is entirely different
|
|
||||||
# We provide a mostly unified API to the user, which needs to be
|
|
||||||
# unfiddled here
|
|
||||||
# ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 )
|
|
||||||
address = tcp.Address((flow.request.host, flow.request.port))
|
|
||||||
|
|
||||||
ssl = (flow.request.scheme == "https")
|
|
||||||
|
|
||||||
if self.c.config.mode == "upstream":
|
|
||||||
# The connection to the upstream proxy may have a state we may need
|
|
||||||
# to take into account.
|
|
||||||
connected_to = None
|
|
||||||
for s in flow.server_conn.state:
|
|
||||||
if s[0] == "http" and s[1]["state"] == "connect":
|
|
||||||
connected_to = tcp.Address((s[1]["host"], s[1]["port"]))
|
|
||||||
|
|
||||||
# We need to reconnect if the current flow either requires a
|
|
||||||
# (possibly impossible) change to the connection state, e.g. the
|
|
||||||
# host has changed but we already CONNECTed somewhere else.
|
|
||||||
needs_server_change = (
|
|
||||||
ssl != self.c.server_conn.ssl_established
|
|
||||||
or
|
|
||||||
# HTTP proxying is "stateless", CONNECT isn't.
|
|
||||||
(connected_to and address != connected_to)
|
|
||||||
)
|
|
||||||
|
|
||||||
if needs_server_change:
|
|
||||||
# force create new connection to the proxy server to reset
|
|
||||||
# state
|
|
||||||
self.live.change_server(self.c.server_conn.address, force=True)
|
|
||||||
if ssl:
|
|
||||||
send_connect_request(
|
|
||||||
self.c.server_conn,
|
|
||||||
address.host,
|
|
||||||
address.port
|
|
||||||
)
|
|
||||||
self.c.establish_ssl(server=True)
|
|
||||||
else:
|
|
||||||
# If we're not in upstream mode, we just want to update the host
|
|
||||||
# and possibly establish TLS. This is a no op if the addresses
|
|
||||||
# match.
|
|
||||||
self.live.change_server(address, ssl=ssl)
|
|
||||||
|
|
||||||
flow.server_conn = self.c.server_conn
|
|
||||||
|
|
||||||
def send_response_to_client(self, flow):
|
|
||||||
if not flow.response.stream:
|
|
||||||
# no streaming:
|
|
||||||
# we already received the full response from the server and can
|
|
||||||
# send it to the client straight away.
|
|
||||||
self.c.client_conn.send(self.c.client_conn.protocol.assemble(flow.response))
|
|
||||||
else:
|
|
||||||
if isinstance(self.c.client_conn.protocol, http2.HTTP2Protocol):
|
|
||||||
raise NotImplementedError("HTTP streaming with HTTP/2 is currently not supported.")
|
|
||||||
|
|
||||||
|
|
||||||
# streaming:
|
|
||||||
# First send the headers and then transfer the response
|
|
||||||
# incrementally:
|
|
||||||
h = self.c.client_conn.protocol._assemble_response_first_line(flow.response)
|
|
||||||
self.c.client_conn.send(h + "\r\n")
|
|
||||||
h = self.c.client_conn.protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True)
|
|
||||||
self.c.client_conn.send(h + "\r\n")
|
|
||||||
|
|
||||||
chunks = self.c.server_conn.protocol.read_http_body_chunked(
|
|
||||||
flow.response.headers,
|
|
||||||
self.c.config.body_size_limit,
|
|
||||||
flow.request.method,
|
|
||||||
flow.response.code,
|
|
||||||
False,
|
|
||||||
4096
|
|
||||||
)
|
|
||||||
|
|
||||||
if callable(flow.response.stream):
|
|
||||||
chunks = flow.response.stream(chunks)
|
|
||||||
|
|
||||||
for chunk in chunks:
|
|
||||||
for part in chunk:
|
|
||||||
self.c.client_conn.wfile.write(part)
|
|
||||||
self.c.client_conn.wfile.flush()
|
|
||||||
|
|
||||||
flow.response.timestamp_end = utils.timestamp()
|
|
||||||
|
|
||||||
def check_close_connection(self, flow):
|
|
||||||
"""
|
|
||||||
Checks if the connection should be closed depending on the HTTP
|
|
||||||
semantics. Returns True, if so.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# TODO: add logic for HTTP/2
|
|
||||||
|
|
||||||
close_connection = (
|
|
||||||
http1.HTTP1Protocol.connection_close(
|
|
||||||
flow.request.httpversion,
|
|
||||||
flow.request.headers
|
|
||||||
) or http1.HTTP1Protocol.connection_close(
|
|
||||||
flow.response.httpversion,
|
|
||||||
flow.response.headers
|
|
||||||
) or http1.HTTP1Protocol.expected_http_body_size(
|
|
||||||
flow.response.headers,
|
|
||||||
False,
|
|
||||||
flow.request.method,
|
|
||||||
flow.response.code) == -1
|
|
||||||
)
|
|
||||||
if close_connection:
|
|
||||||
if flow.request.form_in == "authority" and flow.response.code == 200:
|
|
||||||
# Workaround for
|
|
||||||
# https://github.com/mitmproxy/mitmproxy/issues/313: Some
|
|
||||||
# proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
|
|
||||||
# and no Content-Length header
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def process_connect_request(self, address):
|
|
||||||
"""
|
|
||||||
Process a CONNECT request.
|
|
||||||
Returns True if the CONNECT request has been processed successfully.
|
|
||||||
Returns False, if the connection should be closed immediately.
|
|
||||||
"""
|
|
||||||
address = tcp.Address.wrap(address)
|
|
||||||
if self.c.config.check_ignore(address):
|
|
||||||
self.c.log("Ignore host: %s:%s" % address(), "info")
|
|
||||||
TCPHandler(self.c, log=False).handle_messages()
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
self.expected_form_in = "relative"
|
|
||||||
self.expected_form_out = "relative"
|
|
||||||
self.skip_authentication = True
|
|
||||||
|
|
||||||
# In practice, nobody issues a CONNECT request to send unencrypted
|
|
||||||
# HTTP requests afterwards. If we don't delegate to TCP mode, we
|
|
||||||
# should always negotiate a SSL connection.
|
|
||||||
#
|
|
||||||
# FIXME: Turns out the previous statement isn't entirely true.
|
|
||||||
# Chrome on Windows CONNECTs to :80 if an explicit proxy is
|
|
||||||
# configured and a websocket connection should be established. We
|
|
||||||
# don't support websocket at the moment, so it fails anyway, but we
|
|
||||||
# should come up with a better solution to this if we start to
|
|
||||||
# support WebSockets.
|
|
||||||
should_establish_ssl = (
|
|
||||||
address.port in self.c.config.ssl_ports
|
|
||||||
or
|
|
||||||
not self.c.config.check_tcp(address)
|
|
||||||
)
|
|
||||||
|
|
||||||
if should_establish_ssl:
|
|
||||||
self.c.log(
|
|
||||||
"Received CONNECT request to SSL port. "
|
|
||||||
"Upgrading to SSL...", "debug"
|
|
||||||
)
|
|
||||||
server_ssl = not self.c.config.no_upstream_cert
|
|
||||||
if server_ssl:
|
|
||||||
self.c.establish_server_connection()
|
|
||||||
self.c.establish_ssl(server=server_ssl, client=True)
|
|
||||||
self.c.log("Upgrade to SSL completed.", "debug")
|
|
||||||
|
|
||||||
if self.c.config.check_tcp(address):
|
|
||||||
self.c.log(
|
|
||||||
"Generic TCP mode for host: %s:%s" % address(),
|
|
||||||
"info"
|
|
||||||
)
|
|
||||||
TCPHandler(self.c).handle_messages()
|
|
||||||
return False
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
def authenticate(self, request):
|
|
||||||
if self.c.config.authenticator:
|
|
||||||
if self.c.config.authenticator.authenticate(request.headers):
|
|
||||||
self.c.config.authenticator.clean(request.headers)
|
|
||||||
else:
|
|
||||||
raise http.HttpAuthenticationError(
|
|
||||||
self.c.config.authenticator.auth_challenge_headers())
|
|
||||||
return request.headers
|
|
||||||
@ -1,20 +1,12 @@
|
|||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
import Cookie
|
import Cookie
|
||||||
import copy
|
import copy
|
||||||
import threading
|
|
||||||
import time
|
import time
|
||||||
import urllib
|
|
||||||
import urlparse
|
|
||||||
from email.utils import parsedate_tz, formatdate, mktime_tz
|
from email.utils import parsedate_tz, formatdate, mktime_tz
|
||||||
|
|
||||||
import netlib
|
from netlib import odict, encoding
|
||||||
from netlib import http, tcp, odict, utils, encoding
|
from netlib.http import semantics, CONTENT_MISSING
|
||||||
from netlib.http import cookies, semantics, http1
|
from .. import utils, stateobject
|
||||||
|
|
||||||
from .tcp import TCPHandler
|
|
||||||
from .primitives import KILL, ProtocolHandler, Flow, Error
|
|
||||||
from ..proxy.connection import ServerConnection
|
|
||||||
from .. import utils, controller, stateobject, proxy
|
|
||||||
|
|
||||||
|
|
||||||
class decoded(object):
|
class decoded(object):
|
||||||
@ -170,19 +162,19 @@ class HTTPRequest(MessageMixin, semantics.Request):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
form_in,
|
form_in,
|
||||||
method,
|
method,
|
||||||
scheme,
|
scheme,
|
||||||
host,
|
host,
|
||||||
port,
|
port,
|
||||||
path,
|
path,
|
||||||
httpversion,
|
httpversion,
|
||||||
headers,
|
headers,
|
||||||
body,
|
body,
|
||||||
timestamp_start=None,
|
timestamp_start=None,
|
||||||
timestamp_end=None,
|
timestamp_end=None,
|
||||||
form_out=None,
|
form_out=None,
|
||||||
):
|
):
|
||||||
semantics.Request.__init__(
|
semantics.Request.__init__(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@ -1,11 +1,10 @@
|
|||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
import copy
|
import copy
|
||||||
import uuid
|
import uuid
|
||||||
import netlib.tcp
|
|
||||||
from .. import stateobject, utils, version
|
from .. import stateobject, utils, version
|
||||||
from ..proxy.connection import ClientConnection, ServerConnection
|
from ..proxy.connection import ClientConnection, ServerConnection
|
||||||
|
|
||||||
|
|
||||||
KILL = 0 # const for killed requests
|
KILL = 0 # const for killed requests
|
||||||
|
|
||||||
|
|
||||||
@ -165,130 +164,3 @@ class Flow(stateobject.StateObject):
|
|||||||
self.intercepted = False
|
self.intercepted = False
|
||||||
self.reply()
|
self.reply()
|
||||||
master.handle_accept_intercept(self)
|
master.handle_accept_intercept(self)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class ProtocolHandler(object):
|
|
||||||
"""
|
|
||||||
A ProtocolHandler implements an application-layer protocol, e.g. HTTP.
|
|
||||||
See: libmproxy.protocol.http.HTTPHandler
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, c):
|
|
||||||
self.c = c
|
|
||||||
"""@type: libmproxy.proxy.server.ConnectionHandler"""
|
|
||||||
self.live = LiveConnection(c)
|
|
||||||
"""@type: LiveConnection"""
|
|
||||||
|
|
||||||
def handle_messages(self):
|
|
||||||
"""
|
|
||||||
This method gets called if a client connection has been made. Depending
|
|
||||||
on the proxy settings, a server connection might already exist as well.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError # pragma: nocover
|
|
||||||
|
|
||||||
def handle_server_reconnect(self, state):
|
|
||||||
"""
|
|
||||||
This method gets called if a server connection needs to reconnect and
|
|
||||||
there's a state associated with the server connection (e.g. a
|
|
||||||
previously-sent CONNECT request or a SOCKS proxy request). This method
|
|
||||||
gets called after the connection has been restablished but before SSL is
|
|
||||||
established.
|
|
||||||
"""
|
|
||||||
raise NotImplementedError # pragma: nocover
|
|
||||||
|
|
||||||
def handle_error(self, error):
|
|
||||||
"""
|
|
||||||
This method gets called should there be an uncaught exception during the
|
|
||||||
connection. This might happen outside of handle_messages, e.g. if the
|
|
||||||
initial SSL handshake fails in transparent mode.
|
|
||||||
"""
|
|
||||||
raise error # pragma: nocover
|
|
||||||
|
|
||||||
|
|
||||||
class LiveConnection(object):
|
|
||||||
"""
|
|
||||||
This facade allows interested parties (FlowMaster, inline scripts) to
|
|
||||||
interface with a live connection, without exposing the internals
|
|
||||||
of the ConnectionHandler.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, c):
|
|
||||||
self.c = c
|
|
||||||
"""@type: libmproxy.proxy.server.ConnectionHandler"""
|
|
||||||
self._backup_server_conn = None
|
|
||||||
"""@type: libmproxy.proxy.connection.ServerConnection"""
|
|
||||||
|
|
||||||
def change_server(
|
|
||||||
self,
|
|
||||||
address,
|
|
||||||
ssl=None,
|
|
||||||
sni=None,
|
|
||||||
force=False,
|
|
||||||
persistent_change=False):
|
|
||||||
"""
|
|
||||||
Change the server connection to the specified address.
|
|
||||||
@returns:
|
|
||||||
True, if a new connection has been established,
|
|
||||||
False, if an existing connection has been used
|
|
||||||
"""
|
|
||||||
address = netlib.tcp.Address.wrap(address)
|
|
||||||
|
|
||||||
ssl_mismatch = (
|
|
||||||
ssl is not None and
|
|
||||||
(
|
|
||||||
(self.c.server_conn.connection and ssl != self.c.server_conn.ssl_established)
|
|
||||||
or
|
|
||||||
(sni is not None and sni != self.c.server_conn.sni)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
address_mismatch = (address != self.c.server_conn.address)
|
|
||||||
|
|
||||||
if persistent_change:
|
|
||||||
self._backup_server_conn = None
|
|
||||||
|
|
||||||
if ssl_mismatch or address_mismatch or force:
|
|
||||||
|
|
||||||
self.c.log(
|
|
||||||
"Change server connection: %s:%s -> %s:%s [persistent: %s]" % (
|
|
||||||
self.c.server_conn.address.host,
|
|
||||||
self.c.server_conn.address.port,
|
|
||||||
address.host,
|
|
||||||
address.port,
|
|
||||||
persistent_change
|
|
||||||
),
|
|
||||||
"debug"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not self._backup_server_conn and not persistent_change:
|
|
||||||
self._backup_server_conn = self.c.server_conn
|
|
||||||
self.c.server_conn = None
|
|
||||||
else:
|
|
||||||
# This is at least the second temporary change. We can kill the
|
|
||||||
# current connection.
|
|
||||||
self.c.del_server_connection()
|
|
||||||
|
|
||||||
self.c.set_server_address(address)
|
|
||||||
self.c.establish_server_connection(ask=False)
|
|
||||||
if ssl:
|
|
||||||
self.c.establish_ssl(server=True, sni=sni)
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def restore_server(self):
|
|
||||||
# TODO: Similar to _backup_server_conn, introduce _cache_server_conn,
|
|
||||||
# which keeps the changed connection open This may be beneficial if a
|
|
||||||
# user is rewriting all requests from http to https or similar.
|
|
||||||
if not self._backup_server_conn:
|
|
||||||
return
|
|
||||||
|
|
||||||
self.c.log("Restore original server connection: %s:%s -> %s:%s" % (
|
|
||||||
self.c.server_conn.address.host,
|
|
||||||
self.c.server_conn.address.port,
|
|
||||||
self._backup_server_conn.address.host,
|
|
||||||
self._backup_server_conn.address.port
|
|
||||||
), "debug")
|
|
||||||
|
|
||||||
self.c.del_server_connection()
|
|
||||||
self.c.server_conn = self._backup_server_conn
|
|
||||||
self._backup_server_conn = None
|
|
||||||
|
|||||||
@ -1,97 +0,0 @@
|
|||||||
from __future__ import absolute_import
|
|
||||||
import select
|
|
||||||
import socket
|
|
||||||
from .primitives import ProtocolHandler
|
|
||||||
from netlib.utils import cleanBin
|
|
||||||
from netlib.tcp import NetLibError
|
|
||||||
|
|
||||||
|
|
||||||
class TCPHandler(ProtocolHandler):
|
|
||||||
"""
|
|
||||||
TCPHandler acts as a generic TCP forwarder.
|
|
||||||
Data will be .log()ed, but not stored any further.
|
|
||||||
"""
|
|
||||||
|
|
||||||
chunk_size = 4096
|
|
||||||
|
|
||||||
def __init__(self, c, log=True):
|
|
||||||
super(TCPHandler, self).__init__(c)
|
|
||||||
self.log = log
|
|
||||||
|
|
||||||
def handle_messages(self):
|
|
||||||
self.c.establish_server_connection()
|
|
||||||
|
|
||||||
server = "%s:%s" % self.c.server_conn.address()[:2]
|
|
||||||
buf = memoryview(bytearray(self.chunk_size))
|
|
||||||
conns = [self.c.client_conn.rfile, self.c.server_conn.rfile]
|
|
||||||
|
|
||||||
try:
|
|
||||||
while True:
|
|
||||||
r, _, _ = select.select(conns, [], [], 10)
|
|
||||||
for rfile in r:
|
|
||||||
if self.c.client_conn.rfile == rfile:
|
|
||||||
src, dst = self.c.client_conn, self.c.server_conn
|
|
||||||
direction = "-> tcp ->"
|
|
||||||
src_str, dst_str = "client", server
|
|
||||||
else:
|
|
||||||
dst, src = self.c.client_conn, self.c.server_conn
|
|
||||||
direction = "<- tcp <-"
|
|
||||||
dst_str, src_str = "client", server
|
|
||||||
|
|
||||||
closed = False
|
|
||||||
if src.ssl_established:
|
|
||||||
# Unfortunately, pyOpenSSL lacks a recv_into function.
|
|
||||||
# We need to read a single byte before .pending()
|
|
||||||
# becomes usable
|
|
||||||
contents = src.rfile.read(1)
|
|
||||||
contents += src.rfile.read(src.connection.pending())
|
|
||||||
if not contents:
|
|
||||||
closed = True
|
|
||||||
else:
|
|
||||||
size = src.connection.recv_into(buf)
|
|
||||||
if not size:
|
|
||||||
closed = True
|
|
||||||
|
|
||||||
if closed:
|
|
||||||
conns.remove(src.rfile)
|
|
||||||
# Shutdown connection to the other peer
|
|
||||||
if dst.ssl_established:
|
|
||||||
# We can't half-close a connection, so we just close everything here.
|
|
||||||
# Sockets will be cleaned up on a higher level.
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
dst.connection.shutdown(socket.SHUT_WR)
|
|
||||||
|
|
||||||
if len(conns) == 0:
|
|
||||||
return
|
|
||||||
continue
|
|
||||||
|
|
||||||
if src.ssl_established or dst.ssl_established:
|
|
||||||
# if one of the peers is over SSL, we need to send
|
|
||||||
# bytes/strings
|
|
||||||
if not src.ssl_established:
|
|
||||||
# we revc'd into buf but need bytes/string now.
|
|
||||||
contents = buf[:size].tobytes()
|
|
||||||
if self.log:
|
|
||||||
self.c.log(
|
|
||||||
"%s %s\r\n%s" % (
|
|
||||||
direction, dst_str, cleanBin(contents)
|
|
||||||
),
|
|
||||||
"info"
|
|
||||||
)
|
|
||||||
# Do not use dst.connection.send here, which may raise
|
|
||||||
# OpenSSL-specific errors.
|
|
||||||
dst.send(contents)
|
|
||||||
else:
|
|
||||||
# socket.socket.send supports raw bytearrays/memoryviews
|
|
||||||
if self.log:
|
|
||||||
self.c.log(
|
|
||||||
"%s %s\r\n%s" % (
|
|
||||||
direction, dst_str, cleanBin(buf.tobytes())
|
|
||||||
),
|
|
||||||
"info"
|
|
||||||
)
|
|
||||||
dst.connection.send(buf[:size])
|
|
||||||
except (socket.error, NetLibError) as e:
|
|
||||||
self.c.log("TCP connection closed unexpectedly.", "debug")
|
|
||||||
return
|
|
||||||
Loading…
Reference in New Issue
Block a user