Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-26 10:16:27 +00:00
Code Issues Projects Releases Wiki Activity

adjust to netlib changes

Browse Source
This commit is contained in:
Maximilian Hils 2015-09-16 18:45:22 +02:00
parent 436a9ea839
commit 0af0608978
23 changed files with 211 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/har_extractor.py
View File

@ -128,7 +128,7 @@ def response(context, flow):
request_query_string = [{"name": k, "value": v}
for k, v in flow.request.get_query()]
request_http_version = ".".join([str(v) for v in flow.request.httpversion])
request_http_version = flow.request.httpversion
# Cookies are shaped as tuples by MITMProxy.
request_cookies = [{"name": k.strip(), "value": v[0]}
for k, v in (flow.request.get_cookies() or {}).iteritems()]

8
libmproxy/cmdline.py
View File

@ -103,11 +103,15 @@ def parse_setheader(s):
def parse_server_spec(url):
p = netlib.utils.parse_url(url)
if not p or not p[1] or p[0] not in ("http", "https"):
try:
p = netlib.utils.parse_url(url)
if p[0] not in ("http", "https"):
raise ValueError()
except ValueError:
raise configargparse.ArgumentTypeError(
"Invalid server specification: %s" % url
)
address = Address(p[1:3])
scheme = p[0].lower()
return config.ServerSpec(scheme, address)

2
libmproxy/console/common.py
View File

@ -4,7 +4,7 @@ import urwid
import urwid.util
import os
from netlib.http.semantics import CONTENT_MISSING
from netlib.http import CONTENT_MISSING
import netlib.utils
from .. import utils

2
libmproxy/console/flowview.py
View File

@ -6,7 +6,7 @@ import sys
import urwid
from netlib import odict
from netlib.http.semantics import CONTENT_MISSING, Headers
from netlib.http import CONTENT_MISSING, Headers
from . import common, grideditor, signals, searchable, tabs
from . import flowdetailview
from .. import utils, controller, contentviews

2
libmproxy/dump.py
View File

@ -6,7 +6,7 @@ import traceback
import click
import itertools
from netlib.http.semantics import CONTENT_MISSING
from netlib.http import CONTENT_MISSING
import netlib.utils
from . import flow, filt, contentviews
from .exceptions import ContentViewException

10
libmproxy/exceptions.py
View File

@ -34,11 +34,7 @@ class Socks5Exception(ProtocolException):
pass
class HttpException(ProtocolException):
pass
class InvalidCredentials(HttpException):
class HttpProtocolException(ProtocolException):
pass
@ -48,3 +44,7 @@ class ServerException(ProxyException):
class ContentViewException(ProxyException):
pass
class ReplayException(ProxyException):
pass

12
libmproxy/flow.py
View File

@ -12,7 +12,8 @@ import urlparse
from netlib import wsgi
from netlib.http.semantics import CONTENT_MISSING, Headers
from netlib.exceptions import HttpException
from netlib.http import CONTENT_MISSING, Headers, http1
import netlib.http
from . import controller, tnetstring, filt, script, version
from .onboarding import app
@ -161,9 +162,8 @@ class StreamLargeBodies(object):
def run(self, flow, is_request):
r = flow.request if is_request else flow.response
code = flow.response.code if flow.response else None
expected_size = netlib.http.http1.HTTP1Protocol.expected_http_body_size(
r.headers, is_request, flow.request.method, code
expected_size = http1.expected_http_body_size(
flow.request, flow.response if not is_request else None
)
if not (0 <= expected_size <= self.max_size):
# r.stream may already be a callable, which we want to preserve.
@ -842,7 +842,7 @@ class FlowMaster(controller.Master):
host,
port,
path,
(1, 1),
b"HTTP/1.1",
headers,
None,
None,
@ -1000,7 +1000,7 @@ class FlowMaster(controller.Master):
try:
if self.stream_large_bodies:
self.stream_large_bodies.run(f, False)
except netlib.http.HttpError:
except HttpException:
f.reply(Kill)
return

25
libmproxy/models/http.py
View File

@ -6,18 +6,17 @@ import time
from libmproxy import utils
from netlib import encoding
from netlib.http import status_codes, Headers
from netlib.http import status_codes, Headers, Request, Response, CONTENT_MISSING
from netlib.tcp import Address
from netlib.http.semantics import Request, Response, CONTENT_MISSING
from .. import version, stateobject
from .flow import Flow
class MessageMixin(stateobject.StateObject):
_stateobject_attributes = dict(
httpversion=tuple,
httpversion=bytes,
headers=Headers,
body=str,
body=bytes,
timestamp_start=float,
timestamp_end=float
)
@ -120,7 +119,7 @@ class HTTPRequest(MessageMixin, Request):
path: Path portion of the URL (not present in authority-form)
httpversion: HTTP version tuple, e.g. (1,1)
httpversion: HTTP version, e.g. "HTTP/1.1"
headers: Headers object
@ -186,11 +185,11 @@ class HTTPRequest(MessageMixin, Request):
_stateobject_attributes = MessageMixin._stateobject_attributes.copy()
_stateobject_attributes.update(
form_in=str,
method=str,
scheme=str,
host=str,
method=bytes,
scheme=bytes,
host=bytes,
port=int,
path=str,
path=bytes,
form_out=str,
is_replay=bool
)
@ -267,7 +266,7 @@ class HTTPResponse(MessageMixin, Response):
Exposes the following attributes:
httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0)
httpversion: HTTP version, e.g. "HTTP/1.1"
status_code: HTTP response status code
@ -312,7 +311,7 @@ class HTTPResponse(MessageMixin, Response):
_stateobject_attributes = MessageMixin._stateobject_attributes.copy()
_stateobject_attributes.update(
status_code=int,
msg=str
msg=bytes
)
@classmethod
@ -532,7 +531,7 @@ def make_error_response(status_code, message, headers=None):
)
return HTTPResponse(
(1, 1), # FIXME: Should be a string.
b"HTTP/1.1",
status_code,
response,
headers,
@ -543,7 +542,7 @@ def make_error_response(status_code, message, headers=None):
def make_connect_request(address):
address = Address.wrap(address)
return HTTPRequest(
"authority", "CONNECT", None, address.host, address.port, None, (1, 1),
"authority", "CONNECT", None, address.host, address.port, None, b"HTTP/1.1",
Headers(), ""
)

193
libmproxy/protocol/http.py
View File

@ -6,14 +6,14 @@ import traceback
import six
from netlib import tcp
from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers
from netlib.http.semantics import CONTENT_MISSING
from netlib.exceptions import HttpException, HttpReadDisconnect
from netlib.http import http1, Headers
from netlib.http import CONTENT_MISSING
from netlib.tcp import NetLibError, Address
from netlib.http.http1 import HTTP1Protocol
from netlib.http.http2 import HTTP2Protocol
from netlib.http.http2.connections import HTTP2Protocol
from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame
from .. import utils
from ..exceptions import InvalidCredentials, HttpException, ProtocolException
from ..exceptions import HttpProtocolException, ProtocolException
from ..models import (
HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error
)
@ -45,14 +45,14 @@ class _StreamingHttpLayer(_HttpLayer):
def read_response_headers(self):
raise NotImplementedError
def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
def read_response_body(self, request, response):
raise NotImplementedError()
yield "this is a generator" # pragma: no cover
def read_response(self, request_method):
def read_response(self, request):
response = self.read_response_headers()
response.body = "".join(
self.read_response_body(response.headers, request_method, response.code)
response.body = b"".join(
self.read_response_body(request, response)
)
return response
@ -64,7 +64,7 @@ class _StreamingHttpLayer(_HttpLayer):
def send_response(self, response):
if response.body == CONTENT_MISSING:
raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING")
raise HttpException("Cannot assemble flow with CONTENT_MISSING")
self.send_response_headers(response)
self.send_response_body(response, [response.body])
@ -73,48 +73,31 @@ class Http1Layer(_StreamingHttpLayer):
def __init__(self, ctx, mode):
super(Http1Layer, self).__init__(ctx)
self.mode = mode
self.client_protocol = HTTP1Protocol(self.client_conn)
self.server_protocol = HTTP1Protocol(self.server_conn)
def read_request(self):
return HTTPRequest.from_protocol(
self.client_protocol,
body_size_limit=self.config.body_size_limit
)
req = http1.read_request(self.client_conn.rfile, body_size_limit=self.config.body_size_limit)
return HTTPRequest.wrap(req)
def send_request(self, request):
self.server_conn.send(self.server_protocol.assemble(request))
self.server_conn.wfile.write(http1.assemble_request(request))
self.server_conn.wfile.flush()
def read_response_headers(self):
return HTTPResponse.from_protocol(
self.server_protocol,
request_method=None, # does not matter if we don't read the body.
body_size_limit=self.config.body_size_limit,
include_body=False
)
resp = http1.read_response_head(self.server_conn.rfile)
return HTTPResponse.wrap(resp)
def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
return self.server_protocol.read_http_body_chunked(
headers,
self.config.body_size_limit,
request_method,
response_code,
False,
max_chunk_size
)
def read_response_body(self, request, response):
expected_size = http1.expected_http_body_size(request, response)
return http1.read_body(self.server_conn.rfile, expected_size, self.config.body_size_limit)
def send_response_headers(self, response):
h = self.client_protocol._assemble_response_first_line(response)
self.client_conn.wfile.write(h + "\r\n")
h = self.client_protocol._assemble_response_headers(
response,
preserve_transfer_encoding=True
)
self.client_conn.wfile.write(h + "\r\n")
raw = http1.assemble_response_head(response, preserve_transfer_encoding=True)
self.client_conn.wfile.write(raw)
self.client_conn.wfile.flush()
def send_response_body(self, response, chunks):
if self.client_protocol.has_chunked_encoding(response.headers):
if b"chunked" in response.headers.get(b"transfer-encoding", b"").lower():
# TODO: Move this into netlib.http.http1
chunks = itertools.chain(
(
"{:x}\r\n{}\r\n".format(len(chunk), chunk)
@ -127,36 +110,24 @@ class Http1Layer(_StreamingHttpLayer):
self.client_conn.wfile.flush()
def check_close_connection(self, flow):
close_connection = (
http1.HTTP1Protocol.connection_close(
flow.request.httpversion,
flow.request.headers
) or http1.HTTP1Protocol.connection_close(
flow.response.httpversion,
flow.response.headers
) or http1.HTTP1Protocol.expected_http_body_size(
flow.response.headers,
False,
flow.request.method,
flow.response.code) == -1
request_close = http1.connection_close(
flow.request.httpversion,
flow.request.headers
)
response_close = http1.connection_close(
flow.response.httpversion,
flow.response.headers
)
read_until_eof = http1.expected_http_body_size(flow.request, flow.response) == -1
close_connection = request_close or response_close or read_until_eof
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for
# https://github.com/mitmproxy/mitmproxy/issues/313: Some
# proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
# Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
# Charles Proxy sends a CONNECT response with HTTP/1.0
# and no Content-Length header
return False
return close_connection
def connect(self):
self.ctx.connect()
self.server_protocol = HTTP1Protocol(self.server_conn)
def set_server(self, *args, **kwargs):
self.ctx.set_server(*args, **kwargs)
self.server_protocol = HTTP1Protocol(self.server_conn)
def __call__(self):
layer = HttpLayer(self, self.mode)
layer()
@ -184,10 +155,10 @@ class Http2Layer(_HttpLayer):
# TODO: implement flow control and WINDOW_UPDATE frames
self.server_conn.send(self.server_protocol.assemble(message))
def read_response(self, request_method):
def read_response(self, request):
return HTTPResponse.from_protocol(
self.server_protocol,
request_method=request_method,
request_method=request.method,
body_size_limit=self.config.body_size_limit,
include_body=True,
stream_id=self._stream_id
@ -295,7 +266,7 @@ class UpstreamConnectLayer(Layer):
def _send_connect_request(self):
self.send_request(self.connect_request)
resp = self.read_response("CONNECT")
resp = self.read_response(self.connect_request)
if resp.code != 200:
raise ProtocolException("Reconnect: Upstream server refuses CONNECT request")
@ -337,28 +308,31 @@ class HttpLayer(Layer):
self.__original_server_conn = self.server_conn
while True:
try:
flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
try:
request = self.read_request()
except tcp.NetLibError:
# don't throw an error for disconnects that happen
# before/between requests.
return
request = self.read_request()
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
self.authenticate(request)
if not self.authenticate(request):
return
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except (HttpException, NetLibError) as e:
self.send_error_response(400, repr(e))
six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
try:
flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.form_in == "authority":
self.handle_regular_mode_connect(request)
return
# Make sure that the incoming request matches our expectations
self.validate_request(request)
flow.request = request
self.process_request_hook(flow)
@ -384,30 +358,26 @@ class HttpLayer(Layer):
self.handle_upstream_mode_connect(flow.request.copy())
return
except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e:
error_propagated = False
if flow.request and not flow.response:
except (HttpException, NetLibError) as e:
self.send_error_response(502, repr(e))
if not flow.response:
flow.error = Error(str(e))
self.channel.ask("error", flow)
self.log(traceback.format_exc(), "debug")
error_propagated = True
try:
self.send_response(make_error_response(
getattr(e, "code", 502),
repr(e)
))
except NetLibError:
pass
if not error_propagated:
if isinstance(e, ProtocolException):
six.reraise(ProtocolException, e, sys.exc_info()[2])
else:
six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
return
else:
six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
finally:
flow.live = False
def send_error_response(self, code, message):
try:
response = make_error_response(code, message)
self.send_response(response)
except NetLibError:
pass
def change_upstream_proxy_server(self, address):
# Make set_upstream_proxy_server always available,
# even if there's no UpstreamConnectLayer
@ -435,10 +405,8 @@ class HttpLayer(Layer):
# First send the headers and then transfer the response incrementally
self.send_response_headers(flow.response)
chunks = self.read_response_body(
flow.response.headers,
flow.request.method,
flow.response.code,
max_chunk_size=4096
flow.request,
flow.response
)
if callable(flow.response.stream):
chunks = flow.response.stream(chunks)
@ -451,11 +419,11 @@ class HttpLayer(Layer):
if self.supports_streaming:
flow.response = self.read_response_headers()
else:
flow.response = self.read_response(flow.request.method)
flow.response = self.read_response(flow.request)
try:
get_response()
except (tcp.NetLibError, HttpErrorConnClosed) as v:
except (tcp.NetLibError, HttpException) as v:
self.log(
"server communication error: %s" % repr(v),
level="debug"
@ -485,10 +453,9 @@ class HttpLayer(Layer):
if flow.response.stream:
flow.response.content = CONTENT_MISSING
else:
flow.response.content = "".join(self.read_response_body(
flow.response.headers,
flow.request.method,
flow.response.code
flow.response.content = b"".join(self.read_response_body(
flow.request,
flow.response
))
flow.response.timestamp_end = utils.timestamp()
@ -543,7 +510,7 @@ class HttpLayer(Layer):
if not self.server_conn:
self.connect()
if tls:
raise HttpException("Cannot change scheme in upstream proxy mode.")
raise HttpProtocolException("Cannot change scheme in upstream proxy mode.")
"""
# This is a very ugly (untested) workaround to solve a very ugly problem.
if self.server_conn and self.server_conn.tls_established and not ssl:
@ -561,12 +528,10 @@ class HttpLayer(Layer):
def validate_request(self, request):
if request.form_in == "absolute" and request.scheme != "http":
self.send_response(
make_error_response(400, "Invalid request scheme: %s" % request.scheme))
raise HttpException("Invalid request scheme: %s" % request.scheme)
expected_request_forms = {
"regular": ("absolute",), # an authority request would already be handled.
"regular": ("authority", "absolute",),
"upstream": ("authority", "absolute"),
"transparent": ("relative",)
}
@ -576,10 +541,9 @@ class HttpLayer(Layer):
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
" or ".join(allowed_request_forms), request.form_in
)
self.send_response(make_error_response(400, err_message))
raise HttpException(err_message)
if self.mode == "regular":
if self.mode == "regular" and request.form_in == "absolute":
request.form_out = "relative"
def authenticate(self, request):
@ -592,4 +556,5 @@ class HttpLayer(Layer):
"Proxy Authentication Required",
Headers(**self.config.authenticator.auth_challenge_headers())
))
raise InvalidCredentials("Proxy Authentication Required")
return False
return True

31
libmproxy/protocol/http_replay.py
View File

@ -1,8 +1,9 @@
from __future__ import (absolute_import, print_function, division)
import threading
from libmproxy.exceptions import ReplayException
from netlib.exceptions import HttpException
from netlib.http import http1
from netlib.http import HttpError
from netlib.http.http1 import HTTP1Protocol
from netlib.tcp import NetLibError
from ..controller import Channel
from ..models import Error, HTTPResponse, ServerConnection, make_connect_request
@ -47,13 +48,17 @@ class RequestReplayThread(threading.Thread):
server_address = self.config.upstream_server.address
server = ServerConnection(server_address)
server.connect()
protocol = HTTP1Protocol(server)
if r.scheme == "https":
connect_request = make_connect_request((r.host, r.port))
server.send(protocol.assemble(connect_request))
resp = protocol.read_response("CONNECT")
server.wfile.write(http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(
server.rfile,
connect_request,
body_size_limit=self.config.body_size_limit
)
if resp.code != 200:
raise HttpError(502, "Upstream server refuses CONNECT request")
raise ReplayException("Upstream server refuses CONNECT request")
server.establish_ssl(
self.config.clientcerts,
sni=self.flow.server_conn.sni
@ -65,7 +70,6 @@ class RequestReplayThread(threading.Thread):
server_address = (r.host, r.port)
server = ServerConnection(server_address)
server.connect()
protocol = HTTP1Protocol(server)
if r.scheme == "https":
server.establish_ssl(
self.config.clientcerts,
@ -73,18 +77,19 @@ class RequestReplayThread(threading.Thread):
)
r.form_out = "relative"
server.send(protocol.assemble(r))
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
self.flow.server_conn = server
self.flow.response = HTTPResponse.from_protocol(
protocol,
r.method,
body_size_limit=self.config.body_size_limit,
self.flow.response = http1.read_response(
server.rfile,
r,
body_size_limit=self.config.body_size_limit
)
if self.channel:
response_reply = self.channel.ask("response", self.flow)
if response_reply == Kill:
raise Kill()
except (HttpError, NetLibError) as v:
except (ReplayException, HttpException, NetLibError) as v:
self.flow.error = Error(repr(v))
if self.channel:
self.channel.ask("error", self.flow)

6
libmproxy/protocol/tls.py
View File

@ -7,7 +7,7 @@ from construct import ConstructError
import six
from netlib.tcp import NetLibError, NetLibInvalidCertificateError
from netlib.http.http1 import HTTP1Protocol
from netlib.http import ALPN_PROTO_HTTP1
from ..contrib.tls._constructs import ClientHello
from ..exceptions import ProtocolException, TlsException, ClientHandshakeException
from .base import Layer
@ -367,8 +367,8 @@ class TlsLayer(Layer):
"""
# This gets triggered if we haven't established an upstream connection yet.
default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1
# alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2
default_alpn = ALPN_PROTO_HTTP1
# alpn_preference = ALPN_PROTO_H2
if self.alpn_for_client_connection in options:
choice = bytes(self.alpn_for_client_connection)

7
libmproxy/proxy/root_context.py
View File

@ -5,8 +5,7 @@ import sys
import six
from libmproxy.exceptions import ProtocolException
from netlib.http.http1 import HTTP1Protocol
from netlib.http.http2 import HTTP2Protocol
from netlib.http import ALPN_PROTO_H2, ALPN_PROTO_HTTP1
from netlib.tcp import NetLibError
from ..protocol import (
RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin
@ -85,9 +84,9 @@ class RootContext(object):
# 5. Check for TLS ALPN (HTTP1/HTTP2)
if isinstance(top_layer, TlsLayer):
alpn = top_layer.client_conn.get_alpn_proto_negotiated()
if alpn == HTTP2Protocol.ALPN_PROTO_H2:
if alpn == ALPN_PROTO_H2:
return Http2Layer(top_layer, 'transparent')
if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1:
if alpn == ALPN_PROTO_HTTP1:
return Http1Layer(top_layer, 'transparent')
# 6. Check for raw tcp mode

4
libmproxy/proxy/server.py
View File

@ -6,7 +6,7 @@ import socket
import six
from netlib import tcp
from netlib.http.http1 import HTTP1Protocol
from netlib.http.http1 import assemble_response
from netlib.tcp import NetLibError
from ..exceptions import ProtocolException, ServerException, ClientHandshakeException
from ..protocol import Kill
@ -138,7 +138,7 @@ class ConnectionHandler(object):
# understandable by HTTP clients and humans.
try:
error_response = make_error_response(502, repr(e))
self.client_conn.send(HTTP1Protocol().assemble(error_response))
self.client_conn.send(assemble_response(error_response))
except NetLibError:
pass
except Exception:

6
libmproxy/web/app.py
View File

@ -128,12 +128,10 @@ class FlowHandler(RequestHandler):
if a == "request":
request = flow.request
for k, v in b.iteritems():
if k in ["method", "scheme", "host", "path"]:
if k in ["method", "scheme", "host", "path", "httpversion"]:
setattr(request, k, str(v))
elif k == "port":
request.port = int(v)
elif k == "httpversion":
request.httpversion = tuple(int(x) for x in v)
elif k == "headers":
request.headers.load_state(v)
else:
@ -147,7 +145,7 @@ class FlowHandler(RequestHandler):
elif k == "code":
response.code = int(v)
elif k == "httpversion":
response.httpversion = tuple(int(x) for x in v)
response.httpversion = str(v)
elif k == "headers":
response.headers.load_state(v)
else:

10
test/test_dump.py
View File

@ -4,7 +4,7 @@ from libmproxy.exceptions import ContentViewException
from libmproxy.models import HTTPResponse
import netlib.tutils
from netlib.http.semantics import CONTENT_MISSING
from netlib.http import CONTENT_MISSING
from libmproxy import dump, flow
from libmproxy.proxy import Log
@ -38,13 +38,13 @@ def test_strfuncs():
flow.request.stickycookie = True
flow.client_conn = mock.MagicMock()
flow.client_conn.address.host = "foo"
flow.response = netlib.tutils.tresp(content=CONTENT_MISSING)
flow.response = netlib.tutils.tresp(body=CONTENT_MISSING)
flow.response.is_replay = True
flow.response.code = 300
m.echo_flow(flow)
flow = tutils.tflow(resp=netlib.tutils.tresp("{"))
flow = tutils.tflow(resp=netlib.tutils.tresp(body="{"))
flow.response.headers["content-type"] = "application/json"
flow.response.code = 400
m.echo_flow(flow)
@ -62,14 +62,14 @@ def test_contentview(get_content_view):
class TestDumpMaster:
def _cycle(self, m, content):
f = tutils.tflow(req=netlib.tutils.treq(content))
f = tutils.tflow(req=netlib.tutils.treq(body=content))
l = Log("connect")
l.reply = mock.MagicMock()
m.handle_log(l)
m.handle_clientconnect(f.client_conn)
m.handle_serverconnect(f.server_conn)
m.handle_request(f)
f.response = HTTPResponse.wrap(netlib.tutils.tresp(content))
f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=content))
f = m.handle_response(f)
m.handle_clientdisconnect(f.client_conn)
return f

5
test/test_filt.py
View File

@ -84,7 +84,7 @@ class TestMatching:
"host",
80,
"/path",
(1, 1),
b"HTTP/1.1",
headers,
"content_request",
None,
@ -99,8 +99,7 @@ class TestMatching:
headers = Headers([["header_response", "svalue"]])
f.response = http.HTTPResponse(
(1,
1),
b"HTTP/1.1",
200,
"OK",
headers,

8
test/test_flow.py
View File

@ -8,7 +8,7 @@ import mock
import netlib.utils
from netlib import odict
from netlib.http.semantics import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers
from netlib.http import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers
from libmproxy import filt, protocol, controller, tnetstring, flow
from libmproxy.models import Error, Flow, HTTPRequest, HTTPResponse, HTTPFlow, decoded
from libmproxy.proxy.config import HostMatcher
@ -849,7 +849,7 @@ class TestFlowMaster:
s = flow.State()
f = tutils.tflow()
f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request))
f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request))
pb = [f]
fm = flow.FlowMaster(None, s)
@ -903,7 +903,7 @@ class TestFlowMaster:
def test_server_playback_kill(self):
s = flow.State()
f = tutils.tflow()
f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request))
f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request))
pb = [f]
fm = flow.FlowMaster(None, s)
fm.refresh_server_playback = True
@ -1043,7 +1043,7 @@ class TestRequest:
def test_getset_form_urlencoded(self):
d = odict.ODict([("one", "two"), ("three", "four")])
r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst)))
r = HTTPRequest.wrap(netlib.tutils.treq(body=netlib.utils.urlencode(d.lst)))
r.headers["content-type"] = HDR_FORM_URLENCODED
assert r.get_form_urlencoded() == d

8
test/test_fuzzing.py
View File

@ -17,15 +17,11 @@ class TestFuzzy(tservers.HTTPProxTest):
p = self.pathoc()
assert p.request(req % self.server.port).status_code == 400
def test_invalid_ports(self):
req = 'get:"http://localhost:999999"'
p = self.pathoc()
assert p.request(req).status_code == 400
def test_invalid_ipv6_url(self):
req = 'get:"http://localhost:%s":i13,"["'
p = self.pathoc()
assert p.request(req % self.server.port).status_code == 400
resp = p.request(req % self.server.port)
assert resp.status_code == 400
# def test_invalid_upstream(self):
# req = r"get:'http://localhost:%s/p/200:i10,\x27+\x27'"

58
test/test_protocol_http.py
View File

@ -1,46 +1,36 @@
import cStringIO
from cStringIO import StringIO
from io import BytesIO
from netlib.exceptions import HttpSyntaxException
from mock import MagicMock
from libmproxy.protocol.http import *
import netlib.http
from netlib.http import http1
from netlib.http.semantics import CONTENT_MISSING
from netlib.tutils import treq, raises
import tutils
import tservers
def mock_protocol(data=''):
rfile = cStringIO.StringIO(data)
wfile = cStringIO.StringIO()
return http1.HTTP1Protocol(rfile=rfile, wfile=wfile)
class TestHTTPResponse:
def test_read_from_stringio(self):
s = "HTTP/1.1 200 OK\r\n" \
"Content-Length: 7\r\n" \
"\r\n"\
"content\r\n" \
"HTTP/1.1 204 OK\r\n" \
"\r\n"
protocol = mock_protocol(s)
r = HTTPResponse.from_protocol(protocol, "GET")
assert r.status_code == 200
assert r.content == "content"
assert HTTPResponse.from_protocol(protocol, "GET").status_code == 204
protocol = mock_protocol(s)
# HEAD must not have content by spec. We should leave it on the pipe.
r = HTTPResponse.from_protocol(protocol, "HEAD")
assert r.status_code == 200
assert r.content == ""
tutils.raises(
"Invalid server response: 'content",
HTTPResponse.from_protocol, protocol, "GET"
s = (
b"HTTP/1.1 200 OK\r\n"
b"Content-Length: 7\r\n"
b"\r\n"
b"content\r\n"
b"HTTP/1.1 204 OK\r\n"
b"\r\n"
)
rfile = BytesIO(s)
r = http1.read_response(rfile, treq())
assert r.status_code == 200
assert r.content == b"content"
assert http1.read_response(rfile, treq()).status_code == 204
rfile = BytesIO(s)
# HEAD must not have content by spec. We should leave it on the pipe.
r = http1.read_response(rfile, treq(method=b"HEAD"))
assert r.status_code == 200
assert r.content == b""
with raises(HttpSyntaxException):
http1.read_response(rfile, treq())
class TestHTTPFlow(object):

8
test/test_proxy.py
View File

@ -9,6 +9,7 @@ from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler
import tutils
from libpathod import test
from netlib import http, tcp
from netlib.http import http1
class TestServerConnection:
@ -26,11 +27,10 @@ class TestServerConnection:
f.request.path = "/p/200:da"
# use this protocol just to assemble - not for actual sending
protocol = http.http1.HTTP1Protocol(rfile=sc.rfile)
sc.send(protocol.assemble(f.request))
sc.wfile.write(http1.assemble_request(f.request))
sc.wfile.flush()
protocol = http.http1.HTTP1Protocol(rfile=sc.rfile)
assert protocol.read_response(f.request.method, 1000)
assert http1.read_response(sc.rfile, f.request, 1000)
assert self.d.last_log()
sc.finish()

52
test/test_server.py
View File

@ -1,13 +1,14 @@
import socket
import time
from OpenSSL import SSL
from netlib.exceptions import HttpReadDisconnect, HttpException
from netlib.tcp import Address
import netlib.tutils
from netlib import tcp, http, socks
from netlib.certutils import SSLCert
from netlib.http import authentication
from netlib.http.semantics import CONTENT_MISSING
from netlib.http import authentication, CONTENT_MISSING, http1
from netlib.tutils import raises
from libpathod import pathoc, pathod
from libmproxy.proxy.config import HostMatcher
@ -143,10 +144,9 @@ class TcpMixin:
# mitmproxy responds with bad gateway
assert self.pathod(spec).status_code == 502
self._ignore_on()
tutils.raises(
"invalid server response",
self.pathod,
spec) # pathoc tries to parse answer as HTTP
with raises(HttpException):
self.pathod(spec) # pathoc tries to parse answer as HTTP
self._ignore_off()
def _tcpproxy_on(self):
@ -250,11 +250,6 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
assert p.request(req % self.server2.urlbase)
assert switched(self.proxy.log)
def test_get_connection_err(self):
p = self.pathoc()
ret = p.request("get:'http://localhost:0'")
assert ret.status_code == 502
def test_blank_leading_line(self):
p = self.pathoc()
req = "get:'%s/p/201':i0,'\r\n'"
@ -262,8 +257,8 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
def test_invalid_headers(self):
p = self.pathoc()
req = p.request("get:'http://foo':h':foo'='bar'")
assert req.status_code == 400
resp = p.request("get:'http://foo':h':foo'='bar'")
assert resp.status_code == 400
def test_empty_chunked_content(self):
"""
@ -570,17 +565,23 @@ class TestProxy(tservers.HTTPProxTest):
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect(("localhost", self.proxy.port))
connection.send(
"GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" %
"GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" %
self.server.port)
connection.send("\r\n")
connection.recv(5000)
# a bit hacky: make sure that we don't just read the headers only.
recvd = 0
while recvd < 1024:
recvd += len(connection.recv(5000))
connection.send(
"GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" %
"GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" %
self.server.port)
connection.send("\r\n")
connection.recv(5000)
recvd = 0
while recvd < 1024:
recvd += len(connection.recv(5000))
connection.close()
print(self.master.state.view._list)
first_flow = self.master.state.view[0]
second_flow = self.master.state.view[1]
assert first_flow.server_conn.timestamp_tcp_setup
@ -718,15 +719,12 @@ class TestStreamRequest(tservers.HTTPProxTest):
(self.server.urlbase, spec))
connection.send("\r\n")
protocol = http.http1.HTTP1Protocol(rfile=fconn)
resp = protocol.read_response("GET", None, include_body=False)
resp = http1.read_response_head(fconn)
assert resp.headers["Transfer-Encoding"] == 'chunked'
assert resp.status_code == 200
chunks = list(protocol.read_http_body_chunked(
resp.headers, None, "GET", 200, False
))
chunks = list(http1.read_body(fconn, None))
assert chunks == ["this", "isatest__reachhex"]
connection.close()
@ -743,7 +741,7 @@ class TestFakeResponse(tservers.HTTPProxTest):
def test_fake(self):
f = self.pathod("200")
assert "header_response" in f.headers
assert "header-response" in f.headers
class TestServerConnect(tservers.HTTPProxTest):
@ -766,7 +764,8 @@ class TestKillRequest(tservers.HTTPProxTest):
masterclass = MasterKillRequest
def test_kill(self):
tutils.raises("server disconnect", self.pathod, "200")
with raises(HttpReadDisconnect):
self.pathod("200")
# Nothing should have hit the server
assert not self.server.last_log()
@ -780,7 +779,8 @@ class TestKillResponse(tservers.HTTPProxTest):
masterclass = MasterKillResponse
def test_kill(self):
tutils.raises("server disconnect", self.pathod, "200")
with raises(HttpReadDisconnect):
self.pathod("200")
# The server should have seen a request
assert self.server.last_log()
@ -907,7 +907,7 @@ class TestUpstreamProxySSL(
"""
def handle_request(f):
f.request.httpversion = (1, 0)
f.request.httpversion = b"HTTP/1.1"
del f.request.headers["Content-Length"]
f.reply()

2
test/tutils.py
View File

@ -102,7 +102,7 @@ def tflowview(request_contents=None):
if request_contents is None:
flow = tflow()
else:
flow = tflow(req=netlib.tutils.treq(request_contents))
flow = tflow(req=netlib.tutils.treq(body=request_contents))
fv = FlowView(m, cs, flow)
return fv

6
web/src/js/components/flowview/messages.js
View File

@ -121,7 +121,7 @@ var RequestLine = React.createClass({
render: function () {
var flow = this.props.flow;
var url = flowutils.RequestUtils.pretty_url(flow.request);
var httpver = "HTTP/" + flow.request.httpversion.join(".");
var httpver = flow.request.httpversion;
return <div className="first-line request-line">
<ValueEditor
@ -175,7 +175,7 @@ var RequestLine = React.createClass({
var ResponseLine = React.createClass({
render: function () {
var flow = this.props.flow;
var httpver = "HTTP/" + flow.response.httpversion.join(".");
var httpver = flow.response.httpversion;
return <div className="first-line response-line">
<ValueEditor
ref="httpVersion"
@ -323,4 +323,4 @@ module.exports = {
Request: Request,
Response: Response,
Error: Error
};
};