Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-22 07:08:10 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4427 from mhils/reorg

Browse Source 
Improve module structure
This commit is contained in:
Maximilian Hils 2021-02-05 08:59:34 +01:00 committed by GitHub
commit bc3f39a202
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
82 changed files with 2890 additions and 2899 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/addons/contentview.py
View File

@ -8,7 +8,7 @@ The content view API is explained in the mitmproxy.contentviews module.
from typing import Optional
from mitmproxy import contentviews, flow
from mitmproxy.net import http
from mitmproxy import http
class ViewSwapCase(contentviews.View):

2
examples/addons/http-reply-from-proxy.py
View File

@ -4,7 +4,7 @@ from mitmproxy import http
def request(flow: http.HTTPFlow) -> None:
if flow.request.pretty_url == "http://example.com/path":
flow.response = http.HTTPResponse.make(
flow.response = http.Response.make(
200, # (optional) status code
b"Hello World", # (optional) content
{"Content-Type": "text/html"} # (optional) headers

2
examples/addons/http-trailers.py
View File

@ -8,7 +8,7 @@ body.
"""
from mitmproxy import http
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
def request(flow: http.HTTPFlow):

4
mitmproxy/addons/asgiapp.py
View File

@ -117,7 +117,7 @@ async def serve(app, flow: http.HTTPFlow):
async def send(event):
if event["type"] == "http.response.start":
flow.response = http.HTTPResponse.make(event["status"], b"", event.get("headers", []))
flow.response = http.Response.make(event["status"], b"", event.get("headers", []))
flow.response.decode()
elif event["type"] == "http.response.body":
flow.response.content += event.get("body", b"")
@ -133,7 +133,7 @@ async def serve(app, flow: http.HTTPFlow):
raise RuntimeError(f"no response sent.")
except Exception:
ctx.log.error(f"Error in asgi app:\n{traceback.format_exc(limit=-5)}")
flow.response = http.HTTPResponse.make(500, b"ASGI Error.")
flow.response = http.Response.make(500, b"ASGI Error.")
finally:
flow.reply.commit()
done.set()

3
mitmproxy/addons/clientplayback.py
View File

@ -14,9 +14,10 @@ from mitmproxy.addons.proxyserver import AsyncReply
from mitmproxy.hooks import UpdateHook
from mitmproxy.net import server_spec
from mitmproxy.options import Options
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy import commands, events, layers, server
from mitmproxy.proxy.context import ConnectionState, Context, Server
from mitmproxy.connection import ConnectionState, Server
from mitmproxy.proxy.layer import CommandGenerator
from mitmproxy.utils import asyncio_utils

9
mitmproxy/addons/dumper.py
View File

@ -10,7 +10,6 @@ from mitmproxy import ctx
from mitmproxy import exceptions
from mitmproxy import flowfilter
from mitmproxy import http
from mitmproxy.net import http as net_http
from mitmproxy.tcp import TCPFlow, TCPMessage
from mitmproxy.utils import human
from mitmproxy.utils import strutils
@ -79,7 +78,7 @@ class Dumper:
if self.errfp:
self.errfp.flush()
def _echo_headers(self, headers: net_http.Headers):
def _echo_headers(self, headers: http.Headers):
for k, v in headers.fields:
k = strutils.bytes_to_escaped_str(k)
v = strutils.bytes_to_escaped_str(v)
@ -89,7 +88,7 @@ class Dumper:
)
self.echo(out, ident=4)
def _echo_trailers(self, trailers: Optional[net_http.Headers]):
def _echo_trailers(self, trailers: Optional[http.Headers]):
if not trailers:
return
self.echo(click.style("--- HTTP Trailers", fg="magenta"), ident=4)
@ -97,7 +96,7 @@ class Dumper:
def _echo_message(
self,
message: Union[net_http.Message, TCPMessage, WebSocketMessage],
message: Union[http.Message, TCPMessage, WebSocketMessage],
flow: Union[http.HTTPFlow, TCPFlow, WebSocketFlow]
):
_, lines, error = contentviews.get_message_content_view(
@ -205,7 +204,7 @@ class Dumper:
if not flow.response.is_http2:
reason = flow.response.reason
else:
reason = net_http.status_codes.RESPONSES.get(flow.response.status_code, "")
reason = http.status_codes.RESPONSES.get(flow.response.status_code, "")
reason = click.style(
strutils.escape_control_characters(reason),
fg=code_color,

8
mitmproxy/addons/export.py
View File

@ -12,7 +12,7 @@ from mitmproxy.net.http.http1 import assemble
from mitmproxy.utils import strutils
def cleanup_request(f: flow.Flow) -> http.HTTPRequest:
def cleanup_request(f: flow.Flow) -> http.Request:
if not getattr(f, "request", None):
raise exceptions.CommandError("Can't export flow with no request.")
assert isinstance(f, http.HTTPFlow)
@ -21,7 +21,7 @@ def cleanup_request(f: flow.Flow) -> http.HTTPRequest:
return request
def pop_headers(request: http.HTTPRequest) -> http.HTTPRequest:
def pop_headers(request: http.Request) -> http.Request:
# Remove some headers that are redundant for curl/httpie export
request.headers.pop('content-length')
if request.headers.get("host", "") == request.host:
@ -31,7 +31,7 @@ def pop_headers(request: http.HTTPRequest) -> http.HTTPRequest:
return request
def cleanup_response(f: flow.Flow) -> http.HTTPResponse:
def cleanup_response(f: flow.Flow) -> http.Response:
if not getattr(f, "response", None):
raise exceptions.CommandError("Can't export flow with no response.")
assert isinstance(f, http.HTTPFlow)
@ -40,7 +40,7 @@ def cleanup_response(f: flow.Flow) -> http.HTTPResponse:
return response
def request_content_for_console(request: http.HTTPRequest) -> str:
def request_content_for_console(request: http.Request) -> str:
try:
text = request.get_text(strict=True)
assert text

4
mitmproxy/addons/maplocal.py
View File

@ -139,7 +139,7 @@ class MapLocal:
ctx.log.warn(f"Could not read file: {e}")
continue
flow.response = http.HTTPResponse.make(
flow.response = http.Response.make(
200,
contents,
headers
@ -147,5 +147,5 @@ class MapLocal:
# only set flow.response once, for the first matching rule
return
if all_candidates:
flow.response = http.HTTPResponse.make(404)
flow.response = http.Response.make(404)
ctx.log.info(f"None of the local file candidates exist: {', '.join(str(x) for x in all_candidates)}")

2
mitmproxy/addons/modifyheaders.py
View File

@ -3,7 +3,7 @@ import typing
from pathlib import Path
from mitmproxy import ctx, exceptions, flowfilter, http
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
from mitmproxy.utils import strutils
from mitmproxy.utils.spec import parse_spec

4
mitmproxy/addons/next_layer.py
View File

@ -1,7 +1,7 @@
import re
from typing import Type, Sequence, Union, Tuple, Any, Iterable, Optional, List
from mitmproxy import ctx, exceptions
from mitmproxy import ctx, exceptions, connection
from mitmproxy.net.tls import is_tls_record_magic
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy import context, layer, layers
@ -43,7 +43,7 @@ class NextLayer:
re.compile(x, re.IGNORECASE) for x in ctx.options.allow_hosts
]
def ignore_connection(self, server_address: Optional[context.Address], data_client: bytes) -> Optional[bool]:
def ignore_connection(self, server_address: Optional[connection.Address], data_client: bytes) -> Optional[bool]:
"""
Returns:
True, if the connection should be ignored.

32
mitmproxy/addons/proxyauth.py
View File

@ -7,12 +7,10 @@ from typing import Tuple
import ldap3
import passlib.apache
import mitmproxy.net.http
from mitmproxy import ctx
from mitmproxy import ctx, connection
from mitmproxy import exceptions
from mitmproxy import http
from mitmproxy.net.http import status_codes
from mitmproxy.proxy import context
REALM = "mitmproxy"
@ -49,7 +47,7 @@ class ProxyAuth:
self.singleuser = None
self.ldapconn = None
self.ldapserver = None
self.authenticated: MutableMapping[context.Client, Tuple[str, str]] = weakref.WeakKeyDictionary()
self.authenticated: MutableMapping[connection.Client, Tuple[str, str]] = weakref.WeakKeyDictionary()
"""Contains all connections that are permanently authenticated after an HTTP CONNECT"""
def load(self, loader):
@ -81,17 +79,25 @@ class ProxyAuth:
else:
return 'Authorization'
def auth_required_response(self) -> http.HTTPResponse:
def auth_required_response(self) -> http.Response:
if self.is_proxy_auth():
return http.make_error_response(
status_codes.PROXY_AUTH_REQUIRED,
headers=mitmproxy.net.http.Headers(Proxy_Authenticate=f'Basic realm="{REALM}"'),
)
status_code = status_codes.PROXY_AUTH_REQUIRED
headers = {"Proxy-Authenticate": f'Basic realm="{REALM}"'}
else:
return http.make_error_response(
status_codes.UNAUTHORIZED,
headers=mitmproxy.net.http.Headers(WWW_Authenticate=f'Basic realm="{REALM}"'),
)
status_code = status_codes.UNAUTHORIZED
headers = {"WWW-Authenticate": f'Basic realm="{REALM}"'}
reason = http.status_codes.RESPONSES[status_code]
return http.Response.make(
status_code,
(
f"<html>"
f"<head><title>{status_code} {reason}</title></head>"
f"<body><h1>{status_code} {reason}</h1></body>"
f"</html>"
),
headers
)
def check(self, f: http.HTTPFlow) -> Optional[Tuple[str, str]]:
"""

10
mitmproxy/addons/tlsconfig.py
View File

@ -3,7 +3,7 @@ from pathlib import Path
from typing import List, Optional, TypedDict, Any
from OpenSSL import SSL
from mitmproxy import certs, ctx, exceptions
from mitmproxy import certs, ctx, exceptions, connection
from mitmproxy.net import tls as net_tls
from mitmproxy.options import CONF_BASENAME
from mitmproxy.proxy import context
@ -113,8 +113,8 @@ class TlsConfig:
self.create_proxy_server_ssl_conn(tls_start)
def create_client_proxy_ssl_conn(self, tls_start: tls.TlsStartData) -> None:
client: context.Client = tls_start.context.client
server: context.Server = tls_start.context.server
client: connection.Client = tls_start.context.client
server: connection.Server = tls_start.context.server
entry = self.get_cert(tls_start.context)
@ -149,8 +149,8 @@ class TlsConfig:
tls_start.ssl_conn.set_accept_state()
def create_proxy_server_ssl_conn(self, tls_start: tls.TlsStartData) -> None:
client: context.Client = tls_start.context.client
server: context.Server = tls_start.context.server
client: connection.Client = tls_start.context.client
server: connection.Server = tls_start.context.server
assert server.address
if ctx.options.ssl_insecure:

22
mitmproxy/addons/view.py
View File

@ -15,14 +15,15 @@ import blinker
import sortedcontainers
import mitmproxy.flow
from mitmproxy import flowfilter, hooks
from mitmproxy import exceptions
from mitmproxy import command
from mitmproxy import ctx
from mitmproxy import io
from mitmproxy import exceptions
from mitmproxy import hooks
from mitmproxy import connection
from mitmproxy import flowfilter
from mitmproxy import http
from mitmproxy import io
from mitmproxy import tcp
from mitmproxy.proxy import context
from mitmproxy.utils import human
@ -133,15 +134,15 @@ class View(collections.abc.Sequence):
self.default_order = OrderRequestStart(self)
self.orders = dict(
time = OrderRequestStart(self), method = OrderRequestMethod(self),
url = OrderRequestURL(self), size = OrderKeySize(self),
time=OrderRequestStart(self), method=OrderRequestMethod(self),
url=OrderRequestURL(self), size=OrderKeySize(self),
)
self.order_key = self.default_order
self.order_reversed = False
self.focus_follow = False
self._view = sortedcontainers.SortedListWithKey(
key = self.order_key
key=self.order_key
)
# The sig_view* signals broadcast events that affect the view. That is,
@ -457,12 +458,12 @@ class View(collections.abc.Sequence):
@command.command("view.flows.create")
def create(self, method: str, url: str) -> None:
try:
req = http.HTTPRequest.make(method.upper(), url)
req = http.Request.make(method.upper(), url)
except ValueError as e:
raise exceptions.CommandError("Invalid URL: %s" % e)
c = context.Client(("", 0), ("", 0), req.timestamp_start - 0.0001)
s = context.Server((req.host, req.port))
c = connection.Client(("", 0), ("", 0), req.timestamp_start - 0.0001)
s = connection.Server((req.host, req.port))
f = http.HTTPFlow(c, s)
f.request = req
@ -622,6 +623,7 @@ class Focus:
"""
Tracks a focus element within a View.
"""
def __init__(self, v: View) -> None:
self.view = v
self._flow: typing.Optional[mitmproxy.flow.Flow] = None

374
mitmproxy/connection.py Normal file
View File

@ -0,0 +1,374 @@
import uuid
import warnings
from abc import ABCMeta
from enum import Flag
from typing import Literal, Optional, Sequence, Tuple, Union
from mitmproxy import certs
from mitmproxy.coretypes import serializable
from mitmproxy.net import server_spec
from mitmproxy.utils import human
class ConnectionState(Flag):
"""The current state of the underlying socket."""
CLOSED = 0
CAN_READ = 1
CAN_WRITE = 2
OPEN = CAN_READ | CAN_WRITE
# practically speaking we may have IPv6 addresses with flowinfo and scope_id,
# but type checking isn't good enough to properly handle tuple unions.
# this version at least provides useful type checking messages.
Address = Tuple[str, int]
class Connection(serializable.Serializable, metaclass=ABCMeta):
"""
Base class for client and server connections.
The connection object only exposes metadata about the connection, but not the underlying socket object.
This is intentional, all I/O should be handled by mitmproxy.proxy.server exclusively.
"""
# all connections have a unique id. While
# f.client_conn == f2.client_conn already holds true for live flows (where we have object identity),
# we also want these semantics for recorded flows.
id: str
"""A unique UUID to identify the connection."""
state: ConnectionState
"""The current connection state."""
peername: Optional[Address]
"""The remote's `(ip, port)` tuple for this connection."""
sockname: Optional[Address]
"""Our local `(ip, port)` tuple for this connection."""
error: Optional[str] = None
"""A string describing the connection error."""
tls: bool = False
"""
`True` if TLS should be established, `False` otherwise.
Note that this property only describes if a connection should eventually be protected using TLS.
To check if TLS has already been established, use `Connection.tls_established`.
"""
certificate_list: Sequence[certs.Cert] = ()
"""
The TLS certificate list as sent by the peer.
The first certificate is the end-entity certificate.
> [RFC 8446] Prior to TLS 1.3, "certificate_list" ordering required each
> certificate to certify the one immediately preceding it; however,
> some implementations allowed some flexibility. Servers sometimes
> send both a current and deprecated intermediate for transitional
> purposes, and others are simply configured incorrectly, but these
> cases can nonetheless be validated properly. For maximum
> compatibility, all implementations SHOULD be prepared to handle
> potentially extraneous certificates and arbitrary orderings from any
> TLS version, with the exception of the end-entity certificate which
> MUST be first.
"""
alpn: Optional[bytes] = None
"""The application-layer protocol as negotiated using
[ALPN](https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation)."""
alpn_offers: Sequence[bytes] = ()
"""The ALPN offers as sent in the ClientHello."""
# we may want to add SSL_CIPHER_description here, but that's currently not exposed by cryptography
cipher: Optional[str] = None
"""The active cipher name as returned by OpenSSL's `SSL_CIPHER_get_name`."""
cipher_list: Sequence[str] = ()
"""Ciphers accepted by the proxy server on this connection."""
tls_version: Optional[str] = None
"""The active TLS version."""
sni: Union[str, Literal[True], None]
"""
The [Server Name Indication (SNI)](https://en.wikipedia.org/wiki/Server_Name_Indication) sent in the ClientHello.
For server connections, this value may also be set to `True`, which means "use `Server.address`".
"""
timestamp_end: Optional[float] = None
"""*Timestamp:* Connection has been closed."""
timestamp_tls_setup: Optional[float] = None
"""*Timestamp:* TLS handshake has been completed successfully."""
@property
def connected(self) -> bool:
"""`True` if Connection.state is ConnectionState.OPEN, `False` otherwise. Read-only."""
return self.state is ConnectionState.OPEN
@property
def tls_established(self) -> bool:
"""`True` if TLS has been established, `False` otherwise. Read-only."""
return self.timestamp_tls_setup is not None
def __eq__(self, other):
if isinstance(other, Connection):
return self.id == other.id
return False
def __hash__(self):
return hash(self.id)
def __repr__(self):
attrs = repr({
k: {
"cipher_list": lambda: f"<{len(v)} ciphers>",
"id": lambda: f"…{v[-6:]}"
}.get(k, lambda: v)()
for k, v in self.__dict__.items()
})
return f"{type(self).__name__}({attrs})"
@property
def alpn_proto_negotiated(self) -> Optional[bytes]: # pragma: no cover
"""*Deprecated:* An outdated alias for Connection.alpn."""
warnings.warn("Connection.alpn_proto_negotiated is deprecated, use Connection.alpn instead.",
DeprecationWarning)
return self.alpn
class Client(Connection):
"""A connection between a client and mitmproxy."""
peername: Address
"""The client's address."""
sockname: Address
"""The local address we received this connection on."""
mitmcert: Optional[certs.Cert] = None
"""
The certificate used by mitmproxy to establish TLS with the client.
"""
sni: Union[str, None] = None
"""The Server Name Indication sent by the client."""
timestamp_start: float
"""*Timestamp:* TCP SYN received"""
def __init__(self, peername, sockname, timestamp_start):
self.id = str(uuid.uuid4())
self.peername = peername
self.sockname = sockname
self.timestamp_start = timestamp_start
self.state = ConnectionState.OPEN
def __str__(self):
if self.alpn:
tls_state = f", alpn={self.alpn.decode(errors='replace')}"
elif self.tls_established:
tls_state = ", tls"
else:
tls_state = ""
return f"Client({human.format_address(self.peername)}, state={self.state.name.lower()}{tls_state})"
def get_state(self):
# Important: Retain full compatibility with old proxy core for now!
# This means we need to add all new fields to the old implementation.
return {
'address': self.peername,
'alpn': self.alpn,
'cipher_name': self.cipher,
'id': self.id,
'mitmcert': self.mitmcert.get_state() if self.mitmcert is not None else None,
'sni': self.sni,
'timestamp_end': self.timestamp_end,
'timestamp_start': self.timestamp_start,
'timestamp_tls_setup': self.timestamp_tls_setup,
'tls_established': self.tls_established,
'tls_extensions': [],
'tls_version': self.tls_version,
# only used in sans-io
'state': self.state.value,
'sockname': self.sockname,
'error': self.error,
'tls': self.tls,
'certificate_list': [x.get_state() for x in self.certificate_list],
'alpn_offers': self.alpn_offers,
'cipher_list': self.cipher_list,
}
@classmethod
def from_state(cls, state) -> "Client":
client = Client(
state["address"],
("mitmproxy", 8080),
state["timestamp_start"]
)
client.set_state(state)
return client
def set_state(self, state):
self.peername = tuple(state["address"]) if state["address"] else None
self.alpn = state["alpn"]
self.cipher = state["cipher_name"]
self.id = state["id"]
self.sni = state["sni"]
self.timestamp_end = state["timestamp_end"]
self.timestamp_start = state["timestamp_start"]
self.timestamp_tls_setup = state["timestamp_tls_setup"]
self.tls_version = state["tls_version"]
# only used in sans-io
self.state = ConnectionState(state["state"])
self.sockname = tuple(state["sockname"]) if state["sockname"] else None
self.error = state["error"]
self.tls = state["tls"]
self.certificate_list = [certs.Cert.from_state(x) for x in state["certificate_list"]]
self.mitmcert = certs.Cert.from_state(state["mitmcert"]) if state["mitmcert"] is not None else None
self.alpn_offers = state["alpn_offers"]
self.cipher_list = state["cipher_list"]
@property
def address(self): # pragma: no cover
"""*Deprecated:* An outdated alias for Client.peername."""
warnings.warn("Client.address is deprecated, use Client.peername instead.", DeprecationWarning, stacklevel=2)
return self.peername
@address.setter
def address(self, x): # pragma: no cover
warnings.warn("Client.address is deprecated, use Client.peername instead.", DeprecationWarning, stacklevel=2)
self.peername = x
@property
def cipher_name(self) -> Optional[str]: # pragma: no cover
"""*Deprecated:* An outdated alias for Connection.cipher."""
warnings.warn("Client.cipher_name is deprecated, use Client.cipher instead.", DeprecationWarning, stacklevel=2)
return self.cipher
@property
def clientcert(self) -> Optional[certs.Cert]: # pragma: no cover
"""*Deprecated:* An outdated alias for Connection.certificate_list[0]."""
warnings.warn("Client.clientcert is deprecated, use Client.certificate_list instead.", DeprecationWarning,
stacklevel=2)
if self.certificate_list:
return self.certificate_list[0]
else:
return None
@clientcert.setter
def clientcert(self, val): # pragma: no cover
warnings.warn("Client.clientcert is deprecated, use Client.certificate_list instead.", DeprecationWarning)
if val:
self.certificate_list = [val]
else:
self.certificate_list = []
class Server(Connection):
"""A connection between mitmproxy and an upstream server."""
peername: Optional[Address] = None
"""The server's resolved `(ip, port)` tuple. Will be set during connection establishment."""
sockname: Optional[Address] = None
address: Optional[Address]
"""The server's `(host, port)` address tuple. The host can either be a domain or a plain IP address."""
timestamp_start: Optional[float] = None
"""*Timestamp:* TCP SYN sent."""
timestamp_tcp_setup: Optional[float] = None
"""*Timestamp:* TCP ACK received."""
sni: Union[str, Literal[True], None] = True
via: Optional[server_spec.ServerSpec] = None
"""An optional proxy server specification via which the connection should be established."""
def __init__(self, address: Optional[Address]):
self.id = str(uuid.uuid4())
self.address = address
self.state = ConnectionState.CLOSED
def __str__(self):
if self.alpn:
tls_state = f", alpn={self.alpn.decode(errors='replace')}"
elif self.tls_established:
tls_state = ", tls"
else:
tls_state = ""
if self.sockname:
local_port = f", src_port={self.sockname[1]}"
else:
local_port = ""
return f"Server({human.format_address(self.address)}, state={self.state.name.lower()}{tls_state}{local_port})"
def get_state(self):
return {
'address': self.address,
'alpn': self.alpn,
'id': self.id,
'ip_address': self.peername,
'sni': self.sni,
'source_address': self.sockname,
'timestamp_end': self.timestamp_end,
'timestamp_start': self.timestamp_start,
'timestamp_tcp_setup': self.timestamp_tcp_setup,
'timestamp_tls_setup': self.timestamp_tls_setup,
'tls_established': self.tls_established,
'tls_version': self.tls_version,
'via': None,
# only used in sans-io
'state': self.state.value,
'error': self.error,
'tls': self.tls,
'certificate_list': [x.get_state() for x in self.certificate_list],
'alpn_offers': self.alpn_offers,
'cipher_name': self.cipher,
'cipher_list': self.cipher_list,
'via2': self.via,
}
@classmethod
def from_state(cls, state) -> "Server":
server = Server(None)
server.set_state(state)
return server
def set_state(self, state):
self.address = tuple(state["address"]) if state["address"] else None
self.alpn = state["alpn"]
self.id = state["id"]
self.peername = tuple(state["ip_address"]) if state["ip_address"] else None
self.sni = state["sni"]
self.sockname = tuple(state["source_address"]) if state["source_address"] else None
self.timestamp_end = state["timestamp_end"]
self.timestamp_start = state["timestamp_start"]
self.timestamp_tcp_setup = state["timestamp_tcp_setup"]
self.timestamp_tls_setup = state["timestamp_tls_setup"]
self.tls_version = state["tls_version"]
self.state = ConnectionState(state["state"])
self.error = state["error"]
self.tls = state["tls"]
self.certificate_list = [certs.Cert.from_state(x) for x in state["certificate_list"]]
self.alpn_offers = state["alpn_offers"]
self.cipher = state["cipher_name"]
self.cipher_list = state["cipher_list"]
self.via = state["via2"]
@property
def ip_address(self) -> Optional[Address]: # pragma: no cover
"""*Deprecated:* An outdated alias for `Server.peername`."""
warnings.warn("Server.ip_address is deprecated, use Server.peername instead.", DeprecationWarning, stacklevel=2)
return self.peername
@property
def cert(self) -> Optional[certs.Cert]: # pragma: no cover
"""*Deprecated:* An outdated alias for `Connection.certificate_list[0]`."""
warnings.warn("Server.cert is deprecated, use Server.certificate_list instead.", DeprecationWarning,
stacklevel=2)
if self.certificate_list:
return self.certificate_list[0]
else:
return None
@cert.setter
def cert(self, val): # pragma: no cover
warnings.warn("Server.cert is deprecated, use Server.certificate_list instead.", DeprecationWarning,
stacklevel=2)
if val:
self.certificate_list = [val]
else:
self.certificate_list = []
__all__ = [
"Connection",
"Client",
"Server",
"ConnectionState"
]

2
mitmproxy/contentviews/__init__.py
View File

@ -16,7 +16,7 @@ from typing import List, Union
from typing import Optional
from mitmproxy import flow
from mitmproxy.net import http
from mitmproxy import http
from mitmproxy.utils import strutils
from . import (
auto, raw, hex, json, xml_html, wbxml, javascript, css,

2
mitmproxy/contentviews/base.py
View File

@ -3,7 +3,7 @@ import typing
from abc import ABC, abstractmethod
from mitmproxy import flow
from mitmproxy.net import http
from mitmproxy import http
KEY_MAX = 30

4
mitmproxy/contentviews/multipart.py
View File

@ -1,7 +1,7 @@
from typing import Optional
from mitmproxy.coretypes import multidict
from mitmproxy.net import http
from mitmproxy.net.http import multipart
from . import base
@ -16,7 +16,7 @@ class ViewMultipart(base.View):
def __call__(self, data: bytes, content_type: Optional[str] = None, **metadata):
if content_type is None:
return
v = http.multipart.decode(content_type, data)
v = multipart.decode(content_type, data)
if v:
return "Multipart form", self._format(v)

2
mitmproxy/contentviews/query.py
View File

@ -1,7 +1,7 @@
from typing import Optional
from . import base
from ..net import http
from .. import http
class ViewQuery(base.View):

38
mitmproxy/flow.py
View File

@ -2,11 +2,10 @@ import time
import typing # noqa
import uuid
from mitmproxy import controller
from mitmproxy import controller, connection
from mitmproxy import exceptions
from mitmproxy import stateobject
from mitmproxy import version
from mitmproxy.proxy import context
class Error(stateobject.StateObject):
@ -14,23 +13,21 @@ class Error(stateobject.StateObject):
An Error.
This is distinct from an protocol error response (say, a HTTP code 500),
which is represented by a normal HTTPResponse object. This class is
which is represented by a normal `mitmproxy.http.Response` object. This class is
responsible for indicating errors that fall outside of normal protocol
communications, like interrupted connections, timeouts, protocol errors.
Exposes the following attributes:
msg: Message describing the error
timestamp: Seconds since the epoch
"""
msg: str
"""Message describing the error."""
timestamp: float
"""Unix timestamp"""
KILLED_MESSAGE = "Connection killed."
def __init__(self, msg: str, timestamp=None) -> None:
"""
@type msg: str
@type timestamp: float
"""
def __init__(self, msg: str, timestamp: typing.Optional[float] = None) -> None:
"""Create an error. If no timestamp is passed, the current time is used."""
self.msg = msg
self.timestamp = timestamp or time.time()
@ -55,18 +52,17 @@ class Error(stateobject.StateObject):
class Flow(stateobject.StateObject):
"""
A Flow is a collection of objects representing a single transaction.
This class is usually subclassed for each protocol, e.g. HTTPFlow.
"""
def __init__(
self,
type: str,
client_conn: context.Client,
server_conn: context.Server,
live: bool=None
self,
type: str,
client_conn: connection.Client,
server_conn: connection.Server,
live: bool = None
) -> None:
self.type = type
self.id = str(uuid.uuid4())
@ -85,8 +81,8 @@ class Flow(stateobject.StateObject):
_stateobject_attributes = dict(
id=str,
error=Error,
client_conn=context.Client,
server_conn=context.Server,
client_conn=connection.Client,
server_conn=connection.Server,
type=str,
intercepted=bool,
is_replay=str,

1192
mitmproxy/http.py
View File

File diff suppressed because it is too large Load Diff

0
mitmproxy/net/http/encoding.py → mitmproxy/net/encoding.py
View File

13
mitmproxy/net/http/__init__.py
View File

@ -1,13 +0,0 @@
from mitmproxy.net.http.request import Request
from mitmproxy.net.http.response import Response
from mitmproxy.net.http.message import Message
from mitmproxy.net.http.headers import Headers, parse_content_type
from mitmproxy.net.http import http1, status_codes, multipart
__all__ = [
"Request",
"Response",
"Message",
"Headers", "parse_content_type",
"http1", "status_codes", "multipart",
]

147
mitmproxy/net/http/headers.py
View File

@ -1,153 +1,6 @@
import collections
from typing import Dict, Optional, Tuple
from mitmproxy.coretypes import multidict
from mitmproxy.utils import strutils
# See also: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/
# While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
def _native(x):
return x.decode("utf-8", "surrogateescape")
def _always_bytes(x):
return strutils.always_bytes(x, "utf-8", "surrogateescape")
class Headers(multidict.MultiDict):
"""
Header class which allows both convenient access to individual headers as well as
direct access to the underlying raw data. Provides a full dictionary interface.
Example:
.. code-block:: python
# Create headers with keyword arguments
>>> h = Headers(host="example.com", content_type="application/xml")
# Headers mostly behave like a normal dict.
>>> h["Host"]
"example.com"
# HTTP Headers are case insensitive
>>> h["host"]
"example.com"
# Headers can also be created from a list of raw (header_name, header_value) byte tuples
>>> h = Headers([
(b"Host",b"example.com"),
(b"Accept",b"text/html"),
(b"accept",b"application/xml")
])
# Multiple headers are folded into a single header as per RFC7230
>>> h["Accept"]
"text/html, application/xml"
# Setting a header removes all existing headers with the same name.
>>> h["Accept"] = "application/text"
>>> h["Accept"]
"application/text"
# bytes(h) returns a HTTP1 header block.
>>> print(bytes(h))
Host: example.com
Accept: application/text
# For full control, the raw header fields can be accessed
>>> h.fields
Caveats:
For use with the "Set-Cookie" header, see :py:meth:`get_all`.
"""
def __init__(self, fields=(), **headers):
"""
Args:
fields: (optional) list of ``(name, value)`` header byte tuples,
e.g. ``[(b"Host", b"example.com")]``. All names and values must be bytes.
**headers: Additional headers to set. Will overwrite existing values from `fields`.
For convenience, underscores in header names will be transformed to dashes -
this behaviour does not extend to other methods.
If ``**headers`` contains multiple keys that have equal ``.lower()`` s,
the behavior is undefined.
"""
super().__init__(fields)
for key, value in self.fields:
if not isinstance(key, bytes) or not isinstance(value, bytes):
raise TypeError("Header fields must be bytes.")
# content_type -> content-type
headers = {
_always_bytes(name).replace(b"_", b"-"): _always_bytes(value)
for name, value in headers.items()
}
self.update(headers)
@staticmethod
def _reduce_values(values):
# Headers can be folded
return ", ".join(values)
@staticmethod
def _kconv(key):
# Headers are case-insensitive
return key.lower()
def __bytes__(self):
if self.fields:
return b"\r\n".join(b": ".join(field) for field in self.fields) + b"\r\n"
else:
return b""
def __delitem__(self, key):
key = _always_bytes(key)
super().__delitem__(key)
def __iter__(self):
for x in super().__iter__():
yield _native(x)
def get_all(self, name):
"""
Like :py:meth:`get`, but does not fold multiple headers into a single one.
This is useful for Set-Cookie headers, which do not support folding.
See also: https://tools.ietf.org/html/rfc7230#section-3.2.2
"""
name = _always_bytes(name)
return [
_native(x) for x in
super().get_all(name)
]
def set_all(self, name, values):
"""
Explicitly set multiple headers for the given key.
See: :py:meth:`get_all`
"""
name = _always_bytes(name)
values = [_always_bytes(x) for x in values]
return super().set_all(name, values)
def insert(self, index, key, value):
key = _always_bytes(key)
value = _always_bytes(value)
super().insert(index, key, value)
def items(self, multi=False):
if multi:
return (
(_native(k), _native(v))
for k, v in self.fields
)
else:
return super().items()
def parse_content_type(c: str) -> Optional[Tuple[str, str, Dict[str, str]]]:
"""

19
mitmproxy/net/http/http1/read.py
View File

@ -2,7 +2,8 @@ import re
import time
from typing import List, Tuple, Iterable, Optional
from mitmproxy.net.http import request, response, headers, url
from mitmproxy.http import Request, Headers, Response
from mitmproxy.net.http import url
def get_header_tokens(headers, key):
@ -38,8 +39,8 @@ def connection_close(http_version, headers):
def expected_http_body_size(
request: request.Request,
response: Optional[response.Response] = None,
request: Request,
response: Optional[Response] = None,
expect_continue_as_0: bool = True
):
"""
@ -141,7 +142,7 @@ def _read_response_line(line: bytes) -> Tuple[bytes, int, bytes]:
return http_version, status_code, reason
def _read_headers(lines: Iterable[bytes]) -> headers.Headers:
def _read_headers(lines: Iterable[bytes]) -> Headers:
"""
Read a set of headers.
Stop once a blank line is reached.
@ -168,10 +169,10 @@ def _read_headers(lines: Iterable[bytes]) -> headers.Headers:
ret.append((name, value))
except ValueError:
raise ValueError(f"Invalid header line: {line!r}")
return headers.Headers(ret)
return Headers(ret)
def read_request_head(lines: List[bytes]) -> request.Request:
def read_request_head(lines: List[bytes]) -> Request:
"""
Parse an HTTP request head (request line + headers) from an iterable of lines
@ -187,7 +188,7 @@ def read_request_head(lines: List[bytes]) -> request.Request:
host, port, method, scheme, authority, path, http_version = _read_request_line(lines[0])
headers = _read_headers(lines[1:])
return request.Request(
return Request(
host=host,
port=port,
method=method,
@ -203,7 +204,7 @@ def read_request_head(lines: List[bytes]) -> request.Request:
)
def read_response_head(lines: List[bytes]) -> response.Response:
def read_response_head(lines: List[bytes]) -> Response:
"""
Parse an HTTP response head (response line + headers) from an iterable of lines
@ -219,7 +220,7 @@ def read_response_head(lines: List[bytes]) -> response.Response:
http_version, status_code, reason = _read_response_line(lines[0])
headers = _read_headers(lines[1:])
return response.Response(
return Response(
http_version=http_version,
status_code=status_code,
reason=reason,

281
mitmproxy/net/http/message.py
View File

@ -1,281 +0,0 @@
import re
from dataclasses import dataclass, fields
from typing import Callable, Optional, Union, cast
from mitmproxy.coretypes import serializable
from mitmproxy.net.http import encoding
from mitmproxy.net.http.headers import Headers, assemble_content_type, parse_content_type
from mitmproxy.utils import strutils, typecheck
@dataclass
class MessageData(serializable.Serializable):
http_version: bytes
headers: Headers
content: Optional[bytes]
trailers: Optional[Headers]
timestamp_start: float
timestamp_end: Optional[float]
# noinspection PyUnreachableCode
if __debug__:
def __post_init__(self):
for field in fields(self):
val = getattr(self, field.name)
typecheck.check_option_type(field.name, val, field.type)
def set_state(self, state):
for k, v in state.items():
if k in ("headers", "trailers") and v is not None:
v = Headers.from_state(v)
setattr(self, k, v)
def get_state(self):
state = vars(self).copy()
state["headers"] = state["headers"].get_state()
if state["trailers"] is not None:
state["trailers"] = state["trailers"].get_state()
return state
@classmethod
def from_state(cls, state):
state["headers"] = Headers.from_state(state["headers"])
if state["trailers"] is not None:
state["trailers"] = Headers.from_state(state["trailers"])
return cls(**state)
class Message(serializable.Serializable):
@classmethod
def from_state(cls, state):
return cls(**state)
def get_state(self):
return self.data.get_state()
def set_state(self, state):
self.data.set_state(state)
data: MessageData
stream: Union[Callable, bool] = False
@property
def http_version(self) -> str:
"""
Version string, e.g. "HTTP/1.1"
"""
return self.data.http_version.decode("utf-8", "surrogateescape")
@http_version.setter
def http_version(self, http_version: Union[str, bytes]) -> None:
self.data.http_version = strutils.always_bytes(http_version, "utf-8", "surrogateescape")
@property
def is_http10(self) -> bool:
return self.data.http_version == b"HTTP/1.0"
@property
def is_http11(self) -> bool:
return self.data.http_version == b"HTTP/1.1"
@property
def is_http2(self) -> bool:
return self.data.http_version == b"HTTP/2.0"
@property
def headers(self) -> Headers:
"""
The HTTP headers.
"""
return self.data.headers
@headers.setter
def headers(self, h: Headers) -> None:
self.data.headers = h
@property
def trailers(self) -> Optional[Headers]:
"""
The HTTP trailers.
"""
return self.data.trailers
@trailers.setter
def trailers(self, h: Optional[Headers]) -> None:
self.data.trailers = h
@property
def raw_content(self) -> Optional[bytes]:
"""
The raw (potentially compressed) HTTP message body as bytes.
See also: :py:attr:`content`, :py:class:`text`
"""
return self.data.content
@raw_content.setter
def raw_content(self, content: Optional[bytes]) -> None:
self.data.content = content
def get_content(self, strict: bool = True) -> Optional[bytes]:
"""
The uncompressed HTTP message body as bytes.
Raises:
ValueError, when the HTTP content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
content = encoding.decode(self.raw_content, ce)
# A client may illegally specify a byte -> str encoding here (e.g. utf8)
if isinstance(content, str):
raise ValueError(f"Invalid Content-Encoding: {ce}")
return content
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def set_content(self, value: Optional[bytes]) -> None:
if value is None:
self.raw_content = None
return
if not isinstance(value, bytes):
raise TypeError(
f"Message content must be bytes, not {type(value).__name__}. "
"Please use .text if you want to assign a str."
)
ce = self.headers.get("content-encoding")
try:
self.raw_content = encoding.encode(value, ce or "identity")
except ValueError:
# So we have an invalid content-encoding?
# Let's remove it!
del self.headers["content-encoding"]
self.raw_content = value
self.headers["content-length"] = str(len(self.raw_content))
content = property(get_content, set_content)
@property
def timestamp_start(self) -> float:
"""
First byte timestamp
"""
return self.data.timestamp_start
@timestamp_start.setter
def timestamp_start(self, timestamp_start: float) -> None:
self.data.timestamp_start = timestamp_start
@property
def timestamp_end(self) -> Optional[float]:
"""
Last byte timestamp
"""
return self.data.timestamp_end
@timestamp_end.setter
def timestamp_end(self, timestamp_end: Optional[float]):
self.data.timestamp_end = timestamp_end
def _get_content_type_charset(self) -> Optional[str]:
ct = parse_content_type(self.headers.get("content-type", ""))
if ct:
return ct[2].get("charset")
return None
def _guess_encoding(self, content: bytes = b"") -> str:
enc = self._get_content_type_charset()
if not enc:
if "json" in self.headers.get("content-type", ""):
enc = "utf8"
if not enc:
meta_charset = re.search(rb"""<meta[^>]+charset=['"]?([^'">]+)""", content)
if meta_charset:
enc = meta_charset.group(1).decode("ascii", "ignore")
if not enc:
if "text/css" in self.headers.get("content-type", ""):
# @charset rule must be the very first thing.
css_charset = re.match(rb"""@charset "([^"]+)";""", content)
if css_charset:
enc = css_charset.group(1).decode("ascii", "ignore")
if not enc:
enc = "latin-1"
# Use GB 18030 as the superset of GB2312 and GBK to fix common encoding problems on Chinese websites.
if enc.lower() in ("gb2312", "gbk"):
enc = "gb18030"
return enc
def get_text(self, strict: bool = True) -> Optional[str]:
"""
The uncompressed and decoded HTTP message body as text.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
content = self.get_content(strict)
if content is None:
return None
enc = self._guess_encoding(content)
try:
return cast(str, encoding.decode(content, enc))
except ValueError:
if strict:
raise
return content.decode("utf8", "surrogateescape")
def set_text(self, text: Optional[str]) -> None:
if text is None:
self.content = None
return
enc = self._guess_encoding()
try:
self.content = encoding.encode(text, enc)
except ValueError:
# Fall back to UTF-8 and update the content-type header.
ct = parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
ct[2]["charset"] = "utf-8"
self.headers["content-type"] = assemble_content_type(*ct)
enc = "utf8"
self.content = text.encode(enc, "surrogateescape")
text = property(get_text, set_text)
def decode(self, strict: bool = True) -> None:
"""
Decodes body based on the current Content-Encoding header, then
removes the header. If there is no Content-Encoding header, no
action is taken.
Raises:
ValueError, when the content-encoding is invalid and strict is True.
"""
decoded = self.get_content(strict)
self.headers.pop("content-encoding", None)
self.content = decoded
def encode(self, e: str) -> None:
"""
Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd".
Any existing content-encodings are overwritten,
the content is not decoded beforehand.
Raises:
ValueError, when the specified content-encoding is invalid.
"""
self.headers["content-encoding"] = e
self.content = self.raw_content
if "content-encoding" not in self.headers:
raise ValueError("Invalid content encoding {}".format(repr(e)))

477
mitmproxy/net/http/request.py
View File

@ -1,477 +0,0 @@
import time
import urllib.parse
from dataclasses import dataclass
from typing import Dict, Iterable, Optional, Tuple, Union
import mitmproxy.net.http.url
from mitmproxy.coretypes import multidict
from mitmproxy.net.http import cookies, multipart
from mitmproxy.net.http import message
from mitmproxy.net.http.headers import Headers
from mitmproxy.utils.strutils import always_bytes, always_str
@dataclass
class RequestData(message.MessageData):
host: str
port: int
method: bytes
scheme: bytes
authority: bytes
path: bytes
class Request(message.Message):
"""
An HTTP request.
"""
data: RequestData
def __init__(
self,
host: str,
port: int,
method: bytes,
scheme: bytes,
authority: bytes,
path: bytes,
http_version: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]],
timestamp_start: float,
timestamp_end: Optional[float],
):
# auto-convert invalid types to retain compatibility with older code.
if isinstance(host, bytes):
host = host.decode("idna", "strict")
if isinstance(method, str):
method = method.encode("ascii", "strict")
if isinstance(scheme, str):
scheme = scheme.encode("ascii", "strict")
if isinstance(authority, str):
authority = authority.encode("ascii", "strict")
if isinstance(path, str):
path = path.encode("ascii", "strict")
if isinstance(http_version, str):
http_version = http_version.encode("ascii", "strict")
if isinstance(content, str):
raise ValueError(f"Content must be bytes, not {type(content).__name__}")
if not isinstance(headers, Headers):
headers = Headers(headers)
if trailers is not None and not isinstance(trailers, Headers):
trailers = Headers(trailers)
self.data = RequestData(
host=host,
port=port,
method=method,
scheme=scheme,
authority=authority,
path=path,
http_version=http_version,
headers=headers,
content=content,
trailers=trailers,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
def __repr__(self) -> str:
if self.host and self.port:
hostport = f"{self.host}:{self.port}"
else:
hostport = ""
path = self.path or ""
return f"Request({self.method} {hostport}{path})"
@classmethod
def make(
cls,
method: str,
url: str,
content: Union[bytes, str] = "",
headers: Union[Headers, Dict[Union[str, bytes], Union[str, bytes]], Iterable[Tuple[bytes, bytes]]] = ()
) -> "Request":
"""
Simplified API for creating request objects.
"""
# Headers can be list or dict, we differentiate here.
if isinstance(headers, Headers):
pass
elif isinstance(headers, dict):
headers = Headers(
(always_bytes(k, "utf-8", "surrogateescape"),
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__
))
req = cls(
"",
0,
method.encode("utf-8", "surrogateescape"),
b"",
b"",
b"",
b"HTTP/1.1",
headers,
b"",
None,
time.time(),
time.time(),
)
req.url = url
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
req.content = content
elif isinstance(content, str):
req.text = content
else:
raise TypeError(f"Expected content to be str or bytes, but is {type(content).__name__}.")
return req
@property
def first_line_format(self) -> str:
"""
HTTP request form as defined in `RFC7230 <https://tools.ietf.org/html/rfc7230#section-5.3>`_.
origin-form and asterisk-form are subsumed as "relative".
"""
if self.method == "CONNECT":
return "authority"
elif self.authority:
return "absolute"
else:
return "relative"
@property
def method(self) -> str:
"""
HTTP request method, e.g. "GET".
"""
return self.data.method.decode("utf-8", "surrogateescape").upper()
@method.setter
def method(self, val: Union[str, bytes]) -> None:
self.data.method = always_bytes(val, "utf-8", "surrogateescape")
@property
def scheme(self) -> str:
"""
HTTP request scheme, which should be "http" or "https".
"""
return self.data.scheme.decode("utf-8", "surrogateescape")
@scheme.setter
def scheme(self, val: Union[str, bytes]) -> None:
self.data.scheme = always_bytes(val, "utf-8", "surrogateescape")
@property
def authority(self) -> str:
"""
HTTP request authority.
For HTTP/1, this is the authority portion of the request target
(in either absolute-form or authority-form)
For HTTP/2, this is the :authority pseudo header.
"""
try:
return self.data.authority.decode("idna")
except UnicodeError:
return self.data.authority.decode("utf8", "surrogateescape")
@authority.setter
def authority(self, val: Union[str, bytes]) -> None:
if isinstance(val, str):
try:
val = val.encode("idna", "strict")
except UnicodeError:
val = val.encode("utf8", "surrogateescape") # type: ignore
self.data.authority = val
@property
def host(self) -> str:
"""
Target host. This may be parsed from the raw request
(e.g. from a ``GET http://example.com/ HTTP/1.1`` request line)
or inferred from the proxy mode (e.g. an IP in transparent mode).
Setting the host attribute also updates the host header and authority information, if present.
"""
return self.data.host
@host.setter
def host(self, val: Union[str, bytes]) -> None:
self.data.host = always_str(val, "idna", "strict")
# Update host header
if "Host" in self.data.headers:
self.data.headers["Host"] = val
# Update authority
if self.data.authority:
self.authority = mitmproxy.net.http.url.hostport(self.scheme, self.host, self.port)
@property
def host_header(self) -> Optional[str]:
"""
The request's host/authority header.
This property maps to either ``request.headers["Host"]`` or
``request.authority``, depending on whether it's HTTP/1.x or HTTP/2.0.
"""
if self.is_http2:
return self.authority or self.data.headers.get("Host", None)
else:
return self.data.headers.get("Host", None)
@host_header.setter
def host_header(self, val: Union[None, str, bytes]) -> None:
if val is None:
if self.is_http2:
self.data.authority = b""
self.headers.pop("Host", None)
else:
if self.is_http2:
self.authority = val # type: ignore
if not self.is_http2 or "Host" in self.headers:
# For h2, we only overwrite, but not create, as :authority is the h2 host header.
self.headers["Host"] = val
@property
def port(self) -> int:
"""
Target port
"""
return self.data.port
@port.setter
def port(self, port: int) -> None:
self.data.port = port
@property
def path(self) -> str:
"""
HTTP request path, e.g. "/index.html".
Usually starts with a slash, except for OPTIONS requests, which may just be "*".
"""
return self.data.path.decode("utf-8", "surrogateescape")
@path.setter
def path(self, val: Union[str, bytes]) -> None:
self.data.path = always_bytes(val, "utf-8", "surrogateescape")
@property
def url(self) -> str:
"""
The URL string, constructed from the request's URL components.
"""
if self.first_line_format == "authority":
return f"{self.host}:{self.port}"
return mitmproxy.net.http.url.unparse(self.scheme, self.host, self.port, self.path)
@url.setter
def url(self, val: Union[str, bytes]) -> None:
val = always_str(val, "utf-8", "surrogateescape")
self.scheme, self.host, self.port, self.path = mitmproxy.net.http.url.parse(val)
@property
def pretty_host(self) -> str:
"""
Similar to :py:attr:`host`, but using the host/:authority header as an additional (preferred) data source.
This is useful in transparent mode where :py:attr:`host` is only an IP address,
but may not reflect the actual destination as the Host header could be spoofed.
"""
authority = self.host_header
if authority:
return mitmproxy.net.http.url.parse_authority(authority, check=False)[0]
else:
return self.host
@property
def pretty_url(self) -> str:
"""
Like :py:attr:`url`, but using :py:attr:`pretty_host` instead of :py:attr:`host`.
"""
if self.first_line_format == "authority":
return self.authority
host_header = self.host_header
if not host_header:
return self.url
pretty_host, pretty_port = mitmproxy.net.http.url.parse_authority(host_header, check=False)
pretty_port = pretty_port or mitmproxy.net.http.url.default_port(self.scheme) or 443
return mitmproxy.net.http.url.unparse(self.scheme, pretty_host, pretty_port, self.path)
def _get_query(self):
query = urllib.parse.urlparse(self.url).query
return tuple(mitmproxy.net.http.url.decode(query))
def _set_query(self, query_data):
query = mitmproxy.net.http.url.encode(query_data)
_, _, path, params, _, fragment = urllib.parse.urlparse(self.url)
self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
@property
def query(self) -> multidict.MultiDictView:
"""
The request query string as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
"""
return multidict.MultiDictView(
self._get_query,
self._set_query
)
@query.setter
def query(self, value):
self._set_query(value)
def _get_cookies(self):
h = self.headers.get_all("Cookie")
return tuple(cookies.parse_cookie_headers(h))
def _set_cookies(self, value):
self.headers["cookie"] = cookies.format_cookie_header(value)
@property
def cookies(self) -> multidict.MultiDictView:
"""
The request cookies.
An empty :py:class:`~mitmproxy.net.multidict.MultiDictView` object if the cookie monster ate them all.
"""
return multidict.MultiDictView(
self._get_cookies,
self._set_cookies
)
@cookies.setter
def cookies(self, value):
self._set_cookies(value)
@property
def path_components(self):
"""
The URL's path components as a tuple of strings.
Components are unquoted.
"""
path = urllib.parse.urlparse(self.url).path
# This needs to be a tuple so that it's immutable.
# Otherwise, this would fail silently:
# request.path_components.append("foo")
return tuple(mitmproxy.net.http.url.unquote(i) for i in path.split("/") if i)
@path_components.setter
def path_components(self, components):
components = map(lambda x: mitmproxy.net.http.url.quote(x, safe=""), components)
path = "/" + "/".join(components)
_, _, _, params, query, fragment = urllib.parse.urlparse(self.url)
self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
def anticache(self) -> None:
"""
Modifies this request to remove headers that might produce a cached
response. That is, we remove ETags and If-Modified-Since headers.
"""
delheaders = [
"if-modified-since",
"if-none-match",
]
for i in delheaders:
self.headers.pop(i, None)
def anticomp(self) -> None:
"""
Modifies this request to remove headers that will compress the
resource's data.
"""
self.headers["accept-encoding"] = "identity"
def constrain_encoding(self) -> None:
"""
Limits the permissible Accept-Encoding values, based on what we can
decode appropriately.
"""
accept_encoding = self.headers.get("accept-encoding")
if accept_encoding:
self.headers["accept-encoding"] = (
', '.join(
e
for e in {"gzip", "identity", "deflate", "br", "zstd"}
if e in accept_encoding
)
)
def _get_urlencoded_form(self):
is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
if is_valid_content_type:
return tuple(mitmproxy.net.http.url.decode(self.get_text(strict=False)))
return ()
def _set_urlencoded_form(self, form_data):
"""
Sets the body to the URL-encoded form data, and adds the appropriate content-type header.
This will overwrite the existing content if there is one.
"""
self.headers["content-type"] = "application/x-www-form-urlencoded"
self.content = mitmproxy.net.http.url.encode(form_data, self.get_text(strict=False)).encode()
@property
def urlencoded_form(self):
"""
The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
An empty multidict.MultiDictView if the content-type indicates non-form data
or the content could not be parsed.
Starting with mitmproxy 1.0, key and value are strings.
"""
return multidict.MultiDictView(
self._get_urlencoded_form,
self._set_urlencoded_form
)
@urlencoded_form.setter
def urlencoded_form(self, value):
self._set_urlencoded_form(value)
def _get_multipart_form(self):
is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower()
if is_valid_content_type:
try:
return multipart.decode(self.headers.get("content-type"), self.content)
except ValueError:
pass
return ()
def _set_multipart_form(self, value):
self.content = mitmproxy.net.http.multipart.encode(self.headers, value)
self.headers["content-type"] = "multipart/form-data"
@property
def multipart_form(self):
"""
The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
An empty multidict.MultiDictView if the content-type indicates non-form data
or the content could not be parsed.
Key and value are bytes.
"""
return multidict.MultiDictView(
self._get_multipart_form,
self._set_multipart_form
)
@multipart_form.setter
def multipart_form(self, value):
self._set_multipart_form(value)

211
mitmproxy/net/http/response.py
View File

@ -1,211 +0,0 @@
import time
from dataclasses import dataclass
from email.utils import formatdate, mktime_tz, parsedate_tz
from typing import Mapping
from typing import Iterable
from typing import Optional
from typing import Tuple
from typing import Union
from mitmproxy.coretypes import multidict
from mitmproxy.net.http import cookies, message
from mitmproxy.net.http import status_codes
from mitmproxy.net.http.headers import Headers
from mitmproxy.utils import human
from mitmproxy.utils import strutils
from mitmproxy.utils.strutils import always_bytes
@dataclass
class ResponseData(message.MessageData):
status_code: int
reason: bytes
class Response(message.Message):
"""
An HTTP response.
"""
data: ResponseData
def __init__(
self,
http_version: bytes,
status_code: int,
reason: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]],
timestamp_start: float,
timestamp_end: Optional[float],
):
# auto-convert invalid types to retain compatibility with older code.
if isinstance(http_version, str):
http_version = http_version.encode("ascii", "strict")
if isinstance(reason, str):
reason = reason.encode("ascii", "strict")
if isinstance(content, str):
raise ValueError("Content must be bytes, not {}".format(type(content).__name__))
if not isinstance(headers, Headers):
headers = Headers(headers)
if trailers is not None and not isinstance(trailers, Headers):
trailers = Headers(trailers)
self.data = ResponseData(
http_version=http_version,
status_code=status_code,
reason=reason,
headers=headers,
content=content,
trailers=trailers,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
def __repr__(self) -> str:
if self.raw_content:
ct = self.headers.get("content-type", "unknown content type")
size = human.pretty_size(len(self.raw_content))
details = f"{ct}, {size}"
else:
details = "no content"
return f"Response({self.status_code}, {details})"
@classmethod
def make(
cls,
status_code: int = 200,
content: Union[bytes, str] = b"",
headers: Union[Headers, Mapping[str, Union[str, bytes]], Iterable[Tuple[bytes, bytes]]] = ()
) -> "Response":
"""
Simplified API for creating response objects.
"""
if isinstance(headers, Headers):
headers = headers
elif isinstance(headers, dict):
headers = Headers(
(always_bytes(k, "utf-8", "surrogateescape"),
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__
))
resp = cls(
b"HTTP/1.1",
status_code,
status_codes.RESPONSES.get(status_code, "").encode(),
headers,
None,
None,
time.time(),
time.time(),
)
# Assign this manually to update the content-length header.
if isinstance(content, bytes):
resp.content = content
elif isinstance(content, str):
resp.text = content
else:
raise TypeError(f"Expected content to be str or bytes, but is {type(content).__name__}.")
return resp
@property
def status_code(self) -> int:
"""
HTTP Status Code, e.g. ``200``.
"""
return self.data.status_code
@status_code.setter
def status_code(self, status_code: int) -> None:
self.data.status_code = status_code
@property
def reason(self) -> str:
"""
HTTP Reason Phrase, e.g. "Not Found".
HTTP/2 responses do not contain a reason phrase, an empty string will be returned instead.
"""
# Encoding: http://stackoverflow.com/a/16674906/934719
return self.data.reason.decode("ISO-8859-1")
@reason.setter
def reason(self, reason: Union[str, bytes]) -> None:
self.data.reason = strutils.always_bytes(reason, "ISO-8859-1")
def _get_cookies(self):
h = self.headers.get_all("set-cookie")
all_cookies = cookies.parse_set_cookie_headers(h)
return tuple(
(name, (value, attrs))
for name, value, attrs in all_cookies
)
def _set_cookies(self, value):
cookie_headers = []
for k, v in value:
header = cookies.format_set_cookie_header([(k, v[0], v[1])])
cookie_headers.append(header)
self.headers.set_all("set-cookie", cookie_headers)
@property
def cookies(self) -> multidict.MultiDictView:
"""
The response cookies. A possibly empty
:py:class:`~mitmproxy.net.multidict.MultiDictView`, where the keys are cookie
name strings, and values are (value, attr) tuples. Value is a string,
and attr is an MultiDictView containing cookie attributes. Within
attrs, unary attributes (e.g. HTTPOnly) are indicated by a Null value.
Caveats:
Updating the attr
"""
return multidict.MultiDictView(
self._get_cookies,
self._set_cookies
)
@cookies.setter
def cookies(self, value):
self._set_cookies(value)
def refresh(self, now=None):
"""
This fairly complex and heuristic function refreshes a server
response for replay.
- It adjusts date, expires and last-modified headers.
- It adjusts cookie expiration.
"""
if not now:
now = time.time()
delta = now - self.timestamp_start
refresh_headers = [
"date",
"expires",
"last-modified",
]
for i in refresh_headers:
if i in self.headers:
d = parsedate_tz(self.headers[i])
if d:
new = mktime_tz(d) + delta
self.headers[i] = formatdate(new, usegmt=True)
c = []
for set_cookie_header in self.headers.get_all("set-cookie"):
try:
refreshed = cookies.refresh_set_cookie_header(set_cookie_header, delta)
except ValueError:
refreshed = set_cookie_header
c.append(refreshed)
if c:
self.headers.set_all("set-cookie", c)

2
mitmproxy/proxy/commands.py
View File

@ -9,7 +9,7 @@ The counterpart to commands are events.
from typing import Literal, Union, TYPE_CHECKING
import mitmproxy.hooks
from mitmproxy.proxy.context import Connection, Server
from mitmproxy.connection import Connection, Server
if TYPE_CHECKING:
import mitmproxy.proxy.layer

380
mitmproxy/proxy/context.py
View File

@ -1,394 +1,30 @@
import uuid
import warnings
from abc import ABCMeta
from enum import Flag
from typing import List, Literal, Optional, Sequence, Tuple, Union, TYPE_CHECKING
from typing import List, TYPE_CHECKING
import mitmproxy
from mitmproxy import certs
from mitmproxy.coretypes import serializable
from mitmproxy.net import server_spec
from mitmproxy import connection
from mitmproxy.options import Options
from mitmproxy.utils import human
if TYPE_CHECKING:
import mitmproxy.proxy.layer
class ConnectionState(Flag):
"""The current state of the underlying socket."""
CLOSED = 0
CAN_READ = 1
CAN_WRITE = 2
OPEN = CAN_READ | CAN_WRITE
# practically speaking we may have IPv6 addresses with flowinfo and scope_id,
# but type checking isn't good enough to properly handle tuple unions.
# this version at least provides useful type checking messages.
Address = Tuple[str, int]
class Connection(serializable.Serializable, metaclass=ABCMeta):
"""
Base class for client and server connections.
The connection object only exposes metadata about the connection, but not the underlying socket object.
This is intentional, all I/O should be handled by mitmproxy.proxy.server exclusively.
"""
# all connections have a unique id. While
# f.client_conn == f2.client_conn already holds true for live flows (where we have object identity),
# we also want these semantics for recorded flows.
id: str
"""A unique UUID to identify the connection."""
state: ConnectionState
"""The current connection state."""
peername: Optional[Address]
"""The remote's `(ip, port)` tuple for this connection."""
sockname: Optional[Address]
"""Our local `(ip, port)` tuple for this connection."""
error: Optional[str] = None
"""A string describing the connection error."""
tls: bool = False
"""
`True` if TLS should be established, `False` otherwise.
Note that this property only describes if a connection should eventually be protected using TLS.
To check if TLS has already been established, use `Connection.tls_established`.
"""
certificate_list: Sequence[certs.Cert] = ()
"""
The TLS certificate list as sent by the peer.
The first certificate is the end-entity certificate.
> [RFC 8446] Prior to TLS 1.3, "certificate_list" ordering required each
> certificate to certify the one immediately preceding it; however,
> some implementations allowed some flexibility. Servers sometimes
> send both a current and deprecated intermediate for transitional
> purposes, and others are simply configured incorrectly, but these
> cases can nonetheless be validated properly. For maximum
> compatibility, all implementations SHOULD be prepared to handle
> potentially extraneous certificates and arbitrary orderings from any
> TLS version, with the exception of the end-entity certificate which
> MUST be first.
"""
alpn: Optional[bytes] = None
"""The application-layer protocol as negotiated using
[ALPN](https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation)."""
alpn_offers: Sequence[bytes] = ()
"""The ALPN offers as sent in the ClientHello."""
# we may want to add SSL_CIPHER_description here, but that's currently not exposed by cryptography
cipher: Optional[str] = None
"""The active cipher name as returned by OpenSSL's `SSL_CIPHER_get_name`."""
cipher_list: Sequence[str] = ()
"""Ciphers accepted by the proxy server on this connection."""
tls_version: Optional[str] = None
"""The active TLS version."""
sni: Union[str, Literal[True], None]
"""
The [Server Name Indication (SNI)](https://en.wikipedia.org/wiki/Server_Name_Indication) sent in the ClientHello.
For server connections, this value may also be set to `True`, which means "use `Server.address`".
"""
timestamp_end: Optional[float] = None
"""*Timestamp:* Connection has been closed."""
timestamp_tls_setup: Optional[float] = None
"""*Timestamp:* TLS handshake has been completed successfully."""
@property
def connected(self) -> bool:
"""`True` if Connection.state is ConnectionState.OPEN, `False` otherwise. Read-only."""
return self.state is ConnectionState.OPEN
@property
def tls_established(self) -> bool:
"""`True` if TLS has been established, `False` otherwise. Read-only."""
return self.timestamp_tls_setup is not None
def __eq__(self, other):
if isinstance(other, Connection):
return self.id == other.id
return False
def __hash__(self):
return hash(self.id)
def __repr__(self):
attrs = repr({
k: {
"cipher_list": lambda: f"<{len(v)} ciphers>",
"id": lambda: f"…{v[-6:]}"
}.get(k, lambda: v)()
for k, v in self.__dict__.items()
})
return f"{type(self).__name__}({attrs})"
@property
def alpn_proto_negotiated(self) -> Optional[bytes]: # pragma: no cover
"""*Deprecated:* An outdated alias for Connection.alpn."""
warnings.warn("Connection.alpn_proto_negotiated is deprecated, use Connection.alpn instead.",
DeprecationWarning)
return self.alpn
class Client(Connection):
"""A connection between a client and mitmproxy."""
peername: Address
"""The client's address."""
sockname: Address
"""The local address we received this connection on."""
mitmcert: Optional[certs.Cert] = None
"""
The certificate used by mitmproxy to establish TLS with the client.
"""
sni: Union[str, None] = None
"""The Server Name Indication sent by the client."""
timestamp_start: float
"""*Timestamp:* TCP SYN received"""
def __init__(self, peername, sockname, timestamp_start):
self.id = str(uuid.uuid4())
self.peername = peername
self.sockname = sockname
self.timestamp_start = timestamp_start
self.state = ConnectionState.OPEN
def __str__(self):
if self.alpn:
tls_state = f", alpn={self.alpn.decode(errors='replace')}"
elif self.tls_established:
tls_state = ", tls"
else:
tls_state = ""
return f"Client({human.format_address(self.peername)}, state={self.state.name.lower()}{tls_state})"
def get_state(self):
# Important: Retain full compatibility with old proxy core for now!
# This means we need to add all new fields to the old implementation.
return {
'address': self.peername,
'alpn': self.alpn,
'cipher_name': self.cipher,
'id': self.id,
'mitmcert': self.mitmcert.get_state() if self.mitmcert is not None else None,
'sni': self.sni,
'timestamp_end': self.timestamp_end,
'timestamp_start': self.timestamp_start,
'timestamp_tls_setup': self.timestamp_tls_setup,
'tls_established': self.tls_established,
'tls_extensions': [],
'tls_version': self.tls_version,
# only used in sans-io
'state': self.state.value,
'sockname': self.sockname,
'error': self.error,
'tls': self.tls,
'certificate_list': [x.get_state() for x in self.certificate_list],
'alpn_offers': self.alpn_offers,
'cipher_list': self.cipher_list,
}
@classmethod
def from_state(cls, state) -> "Client":
client = Client(
state["address"],
("mitmproxy", 8080),
state["timestamp_start"]
)
client.set_state(state)
return client
def set_state(self, state):
self.peername = tuple(state["address"]) if state["address"] else None
self.alpn = state["alpn"]
self.cipher = state["cipher_name"]
self.id = state["id"]
self.sni = state["sni"]
self.timestamp_end = state["timestamp_end"]
self.timestamp_start = state["timestamp_start"]
self.timestamp_tls_setup = state["timestamp_tls_setup"]
self.tls_version = state["tls_version"]
# only used in sans-io
self.state = ConnectionState(state["state"])
self.sockname = tuple(state["sockname"]) if state["sockname"] else None
self.error = state["error"]
self.tls = state["tls"]
self.certificate_list = [certs.Cert.from_state(x) for x in state["certificate_list"]]
self.mitmcert = certs.Cert.from_state(state["mitmcert"]) if state["mitmcert"] is not None else None
self.alpn_offers = state["alpn_offers"]
self.cipher_list = state["cipher_list"]
@property
def address(self): # pragma: no cover
"""*Deprecated:* An outdated alias for Client.peername."""
warnings.warn("Client.address is deprecated, use Client.peername instead.", DeprecationWarning, stacklevel=2)
return self.peername
@address.setter
def address(self, x): # pragma: no cover
warnings.warn("Client.address is deprecated, use Client.peername instead.", DeprecationWarning, stacklevel=2)
self.peername = x
@property
def cipher_name(self) -> Optional[str]: # pragma: no cover
"""*Deprecated:* An outdated alias for Connection.cipher."""
warnings.warn("Client.cipher_name is deprecated, use Client.cipher instead.", DeprecationWarning, stacklevel=2)
return self.cipher
@property
def clientcert(self) -> Optional[certs.Cert]: # pragma: no cover
"""*Deprecated:* An outdated alias for Connection.certificate_list[0]."""
warnings.warn("Client.clientcert is deprecated, use Client.certificate_list instead.", DeprecationWarning,
stacklevel=2)
if self.certificate_list:
return self.certificate_list[0]
else:
return None
@clientcert.setter
def clientcert(self, val): # pragma: no cover
warnings.warn("Client.clientcert is deprecated, use Client.certificate_list instead.", DeprecationWarning)
if val:
self.certificate_list = [val]
else:
self.certificate_list = []
class Server(Connection):
"""A connection between mitmproxy and an upstream server."""
peername: Optional[Address] = None
"""The server's resolved `(ip, port)` tuple. Will be set during connection establishment."""
sockname: Optional[Address] = None
address: Optional[Address]
"""The server's `(host, port)` address tuple. The host can either be a domain or a plain IP address."""
timestamp_start: Optional[float] = None
"""*Timestamp:* TCP SYN sent."""
timestamp_tcp_setup: Optional[float] = None
"""*Timestamp:* TCP ACK received."""
sni: Union[str, Literal[True], None] = True
via: Optional[server_spec.ServerSpec] = None
"""An optional proxy server specification via which the connection should be established."""
def __init__(self, address: Optional[Address]):
self.id = str(uuid.uuid4())
self.address = address
self.state = ConnectionState.CLOSED
def __str__(self):
if self.alpn:
tls_state = f", alpn={self.alpn.decode(errors='replace')}"
elif self.tls_established:
tls_state = ", tls"
else:
tls_state = ""
if self.sockname:
local_port = f", src_port={self.sockname[1]}"
else:
local_port = ""
return f"Server({human.format_address(self.address)}, state={self.state.name.lower()}{tls_state}{local_port})"
def get_state(self):
return {
'address': self.address,
'alpn': self.alpn,
'id': self.id,
'ip_address': self.peername,
'sni': self.sni,
'source_address': self.sockname,
'timestamp_end': self.timestamp_end,
'timestamp_start': self.timestamp_start,
'timestamp_tcp_setup': self.timestamp_tcp_setup,
'timestamp_tls_setup': self.timestamp_tls_setup,
'tls_established': self.tls_established,
'tls_version': self.tls_version,
'via': None,
# only used in sans-io
'state': self.state.value,
'error': self.error,
'tls': self.tls,
'certificate_list': [x.get_state() for x in self.certificate_list],
'alpn_offers': self.alpn_offers,
'cipher_name': self.cipher,
'cipher_list': self.cipher_list,
'via2': self.via,
}
@classmethod
def from_state(cls, state) -> "Server":
server = Server(None)
server.set_state(state)
return server
def set_state(self, state):
self.address = tuple(state["address"]) if state["address"] else None
self.alpn = state["alpn"]
self.id = state["id"]
self.peername = tuple(state["ip_address"]) if state["ip_address"] else None
self.sni = state["sni"]
self.sockname = tuple(state["source_address"]) if state["source_address"] else None
self.timestamp_end = state["timestamp_end"]
self.timestamp_start = state["timestamp_start"]
self.timestamp_tcp_setup = state["timestamp_tcp_setup"]
self.timestamp_tls_setup = state["timestamp_tls_setup"]
self.tls_version = state["tls_version"]
self.state = ConnectionState(state["state"])
self.error = state["error"]
self.tls = state["tls"]
self.certificate_list = [certs.Cert.from_state(x) for x in state["certificate_list"]]
self.alpn_offers = state["alpn_offers"]
self.cipher = state["cipher_name"]
self.cipher_list = state["cipher_list"]
self.via = state["via2"]
@property
def ip_address(self) -> Optional[Address]: # pragma: no cover
"""*Deprecated:* An outdated alias for `Server.peername`."""
warnings.warn("Server.ip_address is deprecated, use Server.peername instead.", DeprecationWarning, stacklevel=2)
return self.peername
@property
def cert(self) -> Optional[certs.Cert]: # pragma: no cover
"""*Deprecated:* An outdated alias for `Connection.certificate_list[0]`."""
warnings.warn("Server.cert is deprecated, use Server.certificate_list instead.", DeprecationWarning,
stacklevel=2)
if self.certificate_list:
return self.certificate_list[0]
else:
return None
@cert.setter
def cert(self, val): # pragma: no cover
warnings.warn("Server.cert is deprecated, use Server.certificate_list instead.", DeprecationWarning,
stacklevel=2)
if val:
self.certificate_list = [val]
else:
self.certificate_list = []
class Context:
"""
The context object provided to each `mitmproxy.proxy.layer.Layer` by its parent layer.
"""
client: Client
server: Server
client: connection.Client
server: connection.Server
options: Options
layers: List["mitmproxy.proxy.layer.Layer"]
def __init__(
self,
client: Client,
options: Options,
self,
client: connection.Client,
options: Options,
) -> None:
self.client = client
self.options = options
self.server = Server(None)
self.server = connection.Server(None)
self.layers = []
def fork(self) -> "Context":

2
mitmproxy/proxy/events.py
View File

@ -9,7 +9,7 @@ import warnings
from dataclasses import dataclass, is_dataclass
from mitmproxy.proxy import commands
from mitmproxy.proxy.context import Connection
from mitmproxy.connection import Connection
class Event:

7
mitmproxy/proxy/layer.py
View File

@ -7,9 +7,10 @@ from abc import abstractmethod
from dataclasses import dataclass
from typing import Optional, List, ClassVar, Deque, NamedTuple, Generator, Any, TypeVar
from mitmproxy.connection import Connection
from mitmproxy.proxy import commands, events
from mitmproxy.proxy.commands import Command, StartHook
from mitmproxy.proxy.context import Connection, Context
from mitmproxy.proxy.context import Context
T = TypeVar('T')
CommandGenerator = Generator[Command, Any, T]
@ -96,8 +97,8 @@ class Layer:
if self._paused:
# did we just receive the reply we were waiting for?
pause_finished = (
isinstance(event, events.CommandCompleted) and
event.command is self._paused.command
isinstance(event, events.CommandCompleted) and
event.command is self._paused.command
)
if self.debug is not None:
yield self.__debug(f"{'>>' if pause_finished else '>!'} {event}")

83
mitmproxy/proxy/layers/http/__init__.py
View File

@ -5,15 +5,14 @@ from dataclasses import dataclass
from typing import Optional, Tuple, Union, Dict, DefaultDict, List
from mitmproxy import flow, http
from mitmproxy.connection import Connection, ConnectionState, Server
from mitmproxy.net import server_spec
from mitmproxy.net.http import url
from mitmproxy.proxy import commands, events, layer, tunnel
from mitmproxy.proxy.context import Connection, ConnectionState, Context, Server
from mitmproxy.proxy.layers import tls, websocket, tcp
from mitmproxy.proxy.layers.http import _upstream_proxy
from mitmproxy.proxy.utils import expect
from mitmproxy.utils import human
from ._base import HttpCommand, ReceiveHttp, StreamId, HttpConnection
from ._events import HttpEvent, RequestData, RequestEndOfMessage, RequestHeaders, RequestProtocolError, ResponseData, \
ResponseEndOfMessage, ResponseHeaders, ResponseProtocolError
@ -21,6 +20,7 @@ from ._hooks import HttpConnectHook, HttpErrorHook, HttpRequestHeadersHook, Http
HttpResponseHook
from ._http1 import Http1Client, Http1Server
from ._http2 import Http2Client, Http2Server
from ...context import Context
class HTTPMode(enum.Enum):
@ -55,13 +55,13 @@ class GetHttpConnection(HttpCommand):
def connection_spec_matches(self, connection: Connection) -> bool:
return (
isinstance(connection, Server)
and
self.address == connection.address
and
self.tls == connection.tls
and
self.via == connection.via
isinstance(connection, Server)
and
self.address == connection.address
and
self.tls == connection.tls
and
self.via == connection.via
)
@ -144,7 +144,7 @@ class HttpStream(layer.Layer):
self.flow.request = event.request
if err := validate_request(self.mode, self.flow.request):
self.flow.response = http.HTTPResponse.make(502, str(err))
self.flow.response = http.Response.make(502, str(err))
self.client_state = self.state_errored
return (yield from self.send_response())
@ -162,7 +162,7 @@ class HttpStream(layer.Layer):
try:
host, port = url.parse_authority(self.flow.request.host_header or "", check=True)
except ValueError:
self.flow.response = http.HTTPResponse.make(
self.flow.response = http.Response.make(
400,
"HTTP request has no host header, destination unknown."
)
@ -194,7 +194,7 @@ class HttpStream(layer.Layer):
return
if self.flow.request.headers.get("expect", "").lower() == "100-continue":
continue_response = http.HTTPResponse.make(100)
continue_response = http.Response.make(100)
continue_response.headers.clear()
yield SendHttp(ResponseHeaders(self.stream_id, continue_response), self.context.client)
self.flow.request.headers.pop("expect")
@ -317,9 +317,9 @@ class HttpStream(layer.Layer):
if self.flow.response.status_code == 101:
is_websocket = (
self.flow.response.headers.get("upgrade", "").lower() == "websocket"
and
self.flow.request.headers.get("Sec-WebSocket-Version", "") == "13"
self.flow.response.headers.get("upgrade", "").lower() == "websocket"
and
self.flow.request.headers.get("Sec-WebSocket-Version", "") == "13"
)
if is_websocket and self.context.options.websocket:
self.child_layer = websocket.WebsocketLayer(self.context, self.flow)
@ -338,7 +338,7 @@ class HttpStream(layer.Layer):
def check_killed(self, emit_error_hook: bool) -> layer.CommandGenerator[bool]:
killed_by_us = (
self.flow.error and self.flow.error.msg == flow.Error.KILLED_MESSAGE
self.flow.error and self.flow.error.msg == flow.Error.KILLED_MESSAGE
)
# The client may have closed the connection while we were waiting for the hook to complete.
# We peek into the event queue to see if that is the case.
@ -366,18 +366,18 @@ class HttpStream(layer.Layer):
return False
def handle_protocol_error(
self,
event: Union[RequestProtocolError, ResponseProtocolError]
self,
event: Union[RequestProtocolError, ResponseProtocolError]
) -> layer.CommandGenerator[None]:
is_client_error_but_we_already_talk_upstream = (
isinstance(event, RequestProtocolError)
and self.client_state in (self.state_stream_request_body, self.state_done)
and self.server_state != self.state_errored
isinstance(event, RequestProtocolError)
and self.client_state in (self.state_stream_request_body, self.state_done)
and self.server_state != self.state_errored
)
need_error_hook = not (
self.client_state in (self.state_wait_for_request_headers, self.state_errored)
or
self.server_state in (self.state_done, self.state_errored)
self.client_state in (self.state_wait_for_request_headers, self.state_errored)
or
self.server_state in (self.state_done, self.state_errored)
)
if is_client_error_but_we_already_talk_upstream:
@ -427,7 +427,7 @@ class HttpStream(layer.Layer):
if not self.flow.response and self.context.options.connection_strategy == "eager":
err = yield commands.OpenConnection(self.context.server)
if err:
self.flow.response = http.HTTPResponse.make(
self.flow.response = http.Response.make(
502, f"Cannot connect to {human.format_address(self.context.server.address)}: {err}"
)
self.child_layer = layer.NextLayer(self.context)
@ -449,7 +449,18 @@ class HttpStream(layer.Layer):
def handle_connect_finish(self):
if not self.flow.response:
self.flow.response = http.make_connect_response(self.flow.request.data.http_version)
# Do not send any response headers as it breaks proxying non-80 ports on
# Android emulators using the -http-proxy option.
self.flow.response = http.Response(
self.flow.request.data.http_version,
200,
b"Connection established",
http.Headers(),
b"",
None,
time.time(),
time.time(),
)
if 200 <= self.flow.response.status_code < 300:
yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response), self.context.client)
@ -568,9 +579,9 @@ class HttpLayer(layer.Layer):
raise AssertionError(f"Unexpected event: {event}")
def event_to_child(
self,
child: Union[layer.Layer, HttpStream],
event: events.Event,
self,
child: Union[layer.Layer, HttpStream],
event: events.Event,
) -> layer.CommandGenerator[None]:
for command in child.handle_event(event):
assert isinstance(command, commands.Command)
@ -611,13 +622,13 @@ class HttpLayer(layer.Layer):
for connection in self.connections:
# see "tricky multiplexing edge case" in make_http_connection for an explanation
conn_is_pending_or_h2 = (
connection.alpn == b"h2"
or connection in self.waiting_for_establishment
connection.alpn == b"h2"
or connection in self.waiting_for_establishment
)
h2_to_h1 = self.context.client.alpn == b"h2" and not conn_is_pending_or_h2
connection_suitable = (
event.connection_spec_matches(connection)
and not h2_to_h1
event.connection_spec_matches(connection)
and not h2_to_h1
)
if connection_suitable:
if connection in self.waiting_for_establishment:
@ -628,9 +639,9 @@ class HttpLayer(layer.Layer):
return
can_use_context_connection = (
self.context.server not in self.connections and
self.context.server.connected and
event.connection_spec_matches(self.context.server)
self.context.server not in self.connections and
self.context.server.connected and
event.connection_spec_matches(self.context.server)
)
context = self.context.fork()

3
mitmproxy/proxy/layers/http/_base.py
View File

@ -1,7 +1,8 @@
from dataclasses import dataclass
from mitmproxy.proxy import events, layer, commands
from mitmproxy.proxy.context import Connection, Context
from mitmproxy.connection import Connection
from mitmproxy.proxy.context import Context
StreamId = int

4
mitmproxy/proxy/layers/http/_events.py
View File

@ -8,7 +8,7 @@ from ._base import HttpEvent
@dataclass
class RequestHeaders(HttpEvent):
request: http.HTTPRequest
request: http.Request
end_stream: bool
"""
If True, we already know at this point that there is no message body. This is useful for HTTP/2, where it allows
@ -21,7 +21,7 @@ class RequestHeaders(HttpEvent):
@dataclass
class ResponseHeaders(HttpEvent):
response: http.HTTPResponse
response: http.Response
end_stream: bool = False

60
mitmproxy/proxy/layers/http/_http1.py
View File

@ -1,29 +1,30 @@
import abc
import html
from typing import Union, Optional, Callable, Type
import h11
from h11._readers import ChunkedReader, ContentLengthReader, Http10Reader
from h11._receivebuffer import ReceiveBuffer
from mitmproxy import http
from mitmproxy.net import http as net_http
from mitmproxy import http, version
from mitmproxy.net.http import http1, status_codes
from mitmproxy.proxy import commands, events, layer
from mitmproxy.proxy.context import Connection, ConnectionState, Context
from mitmproxy.connection import Connection, ConnectionState
from mitmproxy.proxy.layers.http._base import ReceiveHttp, StreamId
from mitmproxy.proxy.utils import expect
from mitmproxy.utils import human
from ._base import HttpConnection
from ._events import HttpEvent, RequestData, RequestEndOfMessage, RequestHeaders, RequestProtocolError, ResponseData, \
ResponseEndOfMessage, ResponseHeaders, ResponseProtocolError
from ...context import Context
TBodyReader = Union[ChunkedReader, Http10Reader, ContentLengthReader]
class Http1Connection(HttpConnection, metaclass=abc.ABCMeta):
stream_id: Optional[StreamId] = None
request: Optional[http.HTTPRequest] = None
response: Optional[http.HTTPResponse] = None
request: Optional[http.Request] = None
response: Optional[http.Response] = None
request_done: bool = False
response_done: bool = False
# this is a bit of a hack to make both mypy and PyCharm happy.
@ -146,13 +147,13 @@ class Http1Connection(HttpConnection, metaclass=abc.ABCMeta):
yield from self.make_pipe()
return
connection_done = (
http1.expected_http_body_size(self.request, self.response) == -1
or http1.connection_close(self.request.http_version, self.request.headers)
or http1.connection_close(self.response.http_version, self.response.headers)
# If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request.
# This simplifies our connection management quite a bit as we can rely on
# the proxyserver's max-connection-per-server throttling.
or (self.request.is_http2 and isinstance(self, Http1Client))
http1.expected_http_body_size(self.request, self.response) == -1
or http1.connection_close(self.request.http_version, self.request.headers)
or http1.connection_close(self.response.http_version, self.response.headers)
# If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request.
# This simplifies our connection management quite a bit as we can rely on
# the proxyserver's max-connection-per-server throttling.
or (self.request.is_http2 and isinstance(self, Http1Client))
)
if connection_done:
yield commands.CloseConnection(self.conn)
@ -212,7 +213,7 @@ class Http1Server(Http1Connection):
yield from self.mark_done(response=True)
elif isinstance(event, ResponseProtocolError):
if not self.response:
resp = http.make_error_response(event.code, event.message)
resp = make_error_response(event.code, event.message)
raw = http1.assemble_response(resp)
yield commands.SendData(self.conn, raw)
yield commands.CloseConnection(self.conn)
@ -345,7 +346,7 @@ class Http1Client(Http1Connection):
raise AssertionError(f"Unexpected event: {event}")
def should_make_pipe(request: net_http.Request, response: net_http.Response) -> bool:
def should_make_pipe(request: http.Request, response: http.Response) -> bool:
if response.status_code == 101:
return True
elif response.status_code == 200 and request.method.upper() == "CONNECT":
@ -363,6 +364,37 @@ def make_body_reader(expected_size: Optional[int]) -> TBodyReader:
return ContentLengthReader(expected_size)
def make_error_response(
status_code: int,
message: str = "",
) -> http.Response:
body: bytes = """
<html>
<head>
<title>{status_code} {reason}</title>
</head>
<body>
<h1>{status_code} {reason}</h1>
<p>{message}</p>
</body>
</html>
""".strip().format(
status_code=status_code,
reason=http.status_codes.RESPONSES.get(status_code, "Unknown"),
message=html.escape(message),
).encode("utf8", "replace")
return http.Response.make(
status_code,
body,
http.Headers(
Server=version.MITMPROXY,
Connection="close",
Content_Type="text/html",
)
)
__all__ = [
"Http1Client",
"Http1Server",

36
mitmproxy/proxy/layers/http/_http2.py
View File

@ -13,7 +13,7 @@ import h2.stream
import h2.utilities
from mitmproxy import http
from mitmproxy.net import http as net_http
from mitmproxy.connection import Connection
from mitmproxy.net.http import url, status_codes
from mitmproxy.utils import human
from . import RequestData, RequestEndOfMessage, RequestHeaders, RequestProtocolError, ResponseData, \
@ -21,7 +21,7 @@ from . import RequestData, RequestEndOfMessage, RequestHeaders, RequestProtocolE
from ._base import HttpConnection, HttpEvent, ReceiveHttp
from ._http_h2 import BufferedH2Connection, H2ConnectionLogger
from ...commands import CloseConnection, Log, SendData
from ...context import Connection, Context
from ...context import Context
from ...events import ConnectionClosed, DataReceived, Event, Start
from ...layer import CommandGenerator
from ...utils import expect
@ -197,9 +197,9 @@ class Http2Connection(HttpConnection):
return False
def protocol_error(
self,
message: str,
error_code: int = h2.errors.ErrorCodes.PROTOCOL_ERROR,
self,
message: str,
error_code: int = h2.errors.ErrorCodes.PROTOCOL_ERROR,
) -> CommandGenerator[None]:
yield Log(f"{human.format_address(self.conn.peername)}: {message}")
self.h2_conn.close_connection(error_code, message.encode())
@ -272,7 +272,7 @@ class Http2Server(Http2Connection):
except ValueError as e:
yield from self.protocol_error(f"Invalid HTTP/2 request headers: {e}")
return True
request = http.HTTPRequest(
request = http.Request(
host=host,
port=port,
method=method,
@ -333,8 +333,8 @@ class Http2Client(Http2Connection):
ours = self.our_stream_id.get(event.stream_id, None)
if ours is None:
no_free_streams = (
self.h2_conn.open_outbound_streams >=
(self.provisional_max_concurrency or self.h2_conn.remote_settings.max_concurrent_streams)
self.h2_conn.open_outbound_streams >=
(self.provisional_max_concurrency or self.h2_conn.remote_settings.max_concurrent_streams)
)
if no_free_streams:
self.stream_queue[event.stream_id].append(event)
@ -350,10 +350,10 @@ class Http2Client(Http2Connection):
yield cmd
can_resume_queue = (
self.stream_queue and
self.h2_conn.open_outbound_streams < (
self.provisional_max_concurrency or self.h2_conn.remote_settings.max_concurrent_streams
)
self.stream_queue and
self.h2_conn.open_outbound_streams < (
self.provisional_max_concurrency or self.h2_conn.remote_settings.max_concurrent_streams
)
)
if can_resume_queue:
# popitem would be LIFO, but we want FIFO.
@ -402,7 +402,7 @@ class Http2Client(Http2Connection):
yield from self.protocol_error(f"Invalid HTTP/2 response headers: {e}")
return True
response = http.HTTPResponse(
response = http.Response(
http_version=b"HTTP/2.0",
status_code=status_code,
reason=b"",
@ -427,7 +427,7 @@ class Http2Client(Http2Connection):
return (yield from super().handle_h2_event(event))
def split_pseudo_headers(h2_headers: Sequence[Tuple[bytes, bytes]]) -> Tuple[Dict[bytes, bytes], net_http.Headers]:
def split_pseudo_headers(h2_headers: Sequence[Tuple[bytes, bytes]]) -> Tuple[Dict[bytes, bytes], http.Headers]:
pseudo_headers: Dict[bytes, bytes] = {}
i = 0
for (header, value) in h2_headers:
@ -440,14 +440,14 @@ def split_pseudo_headers(h2_headers: Sequence[Tuple[bytes, bytes]]) -> Tuple[Dic
# Pseudo-headers must be at the start, we are done here.
break
headers = net_http.Headers(h2_headers[i:])
headers = http.Headers(h2_headers[i:])
return pseudo_headers, headers
def parse_h2_request_headers(
h2_headers: Sequence[Tuple[bytes, bytes]]
) -> Tuple[str, int, bytes, bytes, bytes, bytes, net_http.Headers]:
h2_headers: Sequence[Tuple[bytes, bytes]]
) -> Tuple[str, int, bytes, bytes, bytes, bytes, http.Headers]:
"""Split HTTP/2 pseudo-headers from the actual headers and parse them."""
pseudo_headers, headers = split_pseudo_headers(h2_headers)
@ -473,7 +473,7 @@ def parse_h2_request_headers(
return host, port, method, scheme, authority, path, headers
def parse_h2_response_headers(h2_headers: Sequence[Tuple[bytes, bytes]]) -> Tuple[int, net_http.Headers]:
def parse_h2_response_headers(h2_headers: Sequence[Tuple[bytes, bytes]]) -> Tuple[int, http.Headers]:
"""Split HTTP/2 pseudo-headers from the actual headers and parse them."""
pseudo_headers, headers = split_pseudo_headers(h2_headers)

24
mitmproxy/proxy/layers/http/_upstream_proxy.py
View File

@ -1,8 +1,9 @@
import time
from typing import Optional, Tuple
from h11._receivebuffer import ReceiveBuffer
from mitmproxy import http
from mitmproxy import http, connection
from mitmproxy.net import server_spec
from mitmproxy.net.http import http1
from mitmproxy.proxy import commands, context, layer, tunnel
@ -12,13 +13,13 @@ from mitmproxy.utils import human
class HttpUpstreamProxy(tunnel.TunnelLayer):
buf: ReceiveBuffer
send_connect: bool
conn: context.Server
tunnel_connection: context.Server
conn: connection.Server
tunnel_connection: connection.Server
def __init__(
self,
ctx: context.Context,
tunnel_conn: context.Server,
tunnel_conn: connection.Server,
send_connect: bool
):
super().__init__(
@ -44,7 +45,20 @@ class HttpUpstreamProxy(tunnel.TunnelLayer):
if not self.send_connect:
return (yield from super().start_handshake())
assert self.conn.address
req = http.make_connect_request(self.conn.address)
req = http.Request(
host=self.conn.address[0],
port=self.conn.address[1],
method=b"CONNECT",
scheme=b"",
authority=f"{self.conn.address[0]}:{self.conn.address[1]}".encode(),
path=b"",
http_version=b"HTTP/1.1",
headers=http.Headers(),
content=b"",
trailers=None,
timestamp_start=time.time(),
timestamp_end=time.time(),
)
raw = http1.assemble_request(req)
yield commands.SendData(self.tunnel_connection, raw)

3
mitmproxy/proxy/layers/tcp.py
View File

@ -4,7 +4,8 @@ from typing import Optional
from mitmproxy import flow, tcp
from mitmproxy.proxy import commands, events, layer
from mitmproxy.proxy.commands import StartHook
from mitmproxy.proxy.context import ConnectionState, Context, Connection
from mitmproxy.connection import ConnectionState, Connection
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.utils import expect

12
mitmproxy/proxy/layers/tls.py
View File

@ -4,7 +4,7 @@ from dataclasses import dataclass
from typing import Iterator, Optional, Tuple
from OpenSSL import SSL
from mitmproxy import certs
from mitmproxy import certs, connection
from mitmproxy.net import tls as net_tls
from mitmproxy.proxy import commands, events, layer, tunnel
from mitmproxy.proxy import context
@ -116,7 +116,7 @@ class TlsClienthelloHook(StartHook):
@dataclass
class TlsStartData:
conn: context.Connection
conn: connection.Connection
context: context.Context
ssl_conn: Optional[SSL.Connection] = None
@ -136,7 +136,7 @@ class _TLSLayer(tunnel.TunnelLayer):
tls: SSL.Connection = None
"""The OpenSSL connection object"""
def __init__(self, context: context.Context, conn: context.Connection):
def __init__(self, context: context.Context, conn: connection.Connection):
super().__init__(
context,
tunnel_connection=conn,
@ -240,7 +240,7 @@ class _TLSLayer(tunnel.TunnelLayer):
events.DataReceived(self.conn, bytes(plaintext))
)
if close:
self.conn.state &= ~context.ConnectionState.CAN_READ
self.conn.state &= ~connection.ConnectionState.CAN_READ
if self.debug:
yield commands.Log(f"{self.debug}[tls] close_notify {self.conn}", level="debug")
yield from self.event_to_child(
@ -268,7 +268,7 @@ class ServerTLSLayer(_TLSLayer):
"""
command_to_reply_to: Optional[commands.OpenConnection] = None
def __init__(self, context: context.Context, conn: Optional[context.Server] = None):
def __init__(self, context: context.Context, conn: Optional[connection.Server] = None):
super().__init__(context, conn or context.server)
def start_handshake(self) -> layer.CommandGenerator[None]:
@ -373,4 +373,4 @@ class MockTLSLayer(_TLSLayer):
"""
def __init__(self, ctx: context.Context):
super().__init__(ctx, context.Server(None))
super().__init__(ctx, connection.Server(None))

8
mitmproxy/proxy/layers/websocket.py
View File

@ -5,8 +5,8 @@ import wsproto
import wsproto.extensions
import wsproto.frame_protocol
import wsproto.utilities
from mitmproxy import flow, websocket, http
from mitmproxy.proxy import commands, events, layer, context
from mitmproxy import flow, websocket, http, connection
from mitmproxy.proxy import commands, events, layer
from mitmproxy.proxy.commands import StartHook
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.utils import expect
@ -60,10 +60,10 @@ class WebsocketConnection(wsproto.Connection):
- we add a framebuffer for incomplete messages
- we wrap .send() so that we can directly yield it.
"""
conn: context.Connection
conn: connection.Connection
frame_buf: List[Union[str, bytes]]
def __init__(self, *args, conn: context.Connection, **kwargs):
def __init__(self, *args, conn: connection.Connection, **kwargs):
super(WebsocketConnection, self).__init__(*args, **kwargs)
self.conn = conn
self.frame_buf = []

5
mitmproxy/proxy/server.py
View File

@ -17,9 +17,10 @@ from dataclasses import dataclass
from OpenSSL import SSL
from mitmproxy import http, options as moptions
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy import commands, events, layer, layers, server_hooks
from mitmproxy.proxy.context import Address, Client, Connection, ConnectionState, Context
from mitmproxy.connection import Address, Client, Connection, ConnectionState
from mitmproxy.proxy.layers import tls
from mitmproxy.utils import asyncio_utils
from mitmproxy.utils import human
@ -401,7 +402,7 @@ if __name__ == "__main__": # pragma: no cover
def request(flow: http.HTTPFlow):
if "cached" in flow.request.path:
flow.response = http.HTTPResponse.make(418, f"(cached) {flow.request.text}")
flow.response = http.Response.make(418, f"(cached) {flow.request.text}")
if "toggle-tls" in flow.request.path:
if flow.request.url.startswith("https://"):
flow.request.url = flow.request.url.replace("https://", "http://")

11
mitmproxy/proxy/server_hooks.py
View File

@ -1,6 +1,7 @@
from dataclasses import dataclass
from . import commands, context
from mitmproxy import connection
from . import commands
@dataclass
@ -11,7 +12,7 @@ class ClientConnectedHook(commands.StartHook):
Setting client.error kills the connection.
"""
client: context.Client
client: connection.Client
@dataclass
@ -20,13 +21,13 @@ class ClientDisconnectedHook(commands.StartHook):
A client connection has been closed (either by us or the client).
"""
blocking = False
client: context.Client
client: connection.Client
@dataclass
class ServerConnectionHookData:
server: context.Server
client: context.Client
server: connection.Server
client: connection.Client
@dataclass

21
mitmproxy/proxy/tunnel.py
View File

@ -1,6 +1,7 @@
from enum import Enum, auto
from typing import List, Optional, Tuple
from mitmproxy import connection
from mitmproxy.proxy import commands, context, events, layer
from mitmproxy.proxy.layer import Layer
@ -18,9 +19,9 @@ class TunnelLayer(layer.Layer):
or TLS.
"""
child_layer: layer.Layer
tunnel_connection: context.Connection
tunnel_connection: connection.Connection
"""The 'outer' connection which provides the tunnel protocol I/O"""
conn: context.Connection
conn: connection.Connection
"""The 'inner' connection which provides data I/O"""
tunnel_state: TunnelState = TunnelState.INACTIVE
command_to_reply_to: Optional[commands.OpenConnection] = None
@ -33,8 +34,8 @@ class TunnelLayer(layer.Layer):
def __init__(
self,
context: context.Context,
tunnel_connection: context.Connection,
conn: context.Connection,
tunnel_connection: connection.Connection,
conn: connection.Connection,
):
super().__init__(context)
self.tunnel_connection = tunnel_connection
@ -47,7 +48,7 @@ class TunnelLayer(layer.Layer):
def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]:
if isinstance(event, events.Start):
if self.tunnel_connection.state is not context.ConnectionState.CLOSED:
if self.tunnel_connection.state is not connection.ConnectionState.CLOSED:
# we might be in the interesting state here where the connection is already half-closed,
# for example because next_layer buffered events and the client disconnected in the meantime.
# we still expect a close event to arrive, so we carry on here as normal for now.
@ -60,17 +61,17 @@ class TunnelLayer(layer.Layer):
done, err = yield from self.receive_handshake_data(event.data)
if done:
if self.conn != self.tunnel_connection:
self.conn.state = context.ConnectionState.OPEN
self.conn.state = connection.ConnectionState.OPEN
if err:
if self.conn != self.tunnel_connection:
self.conn.state = context.ConnectionState.CLOSED
self.conn.state = connection.ConnectionState.CLOSED
yield from self.on_handshake_error(err)
if done or err:
yield from self._handshake_finished(err)
else:
yield from self.receive_data(event.data)
elif isinstance(event, events.ConnectionClosed):
self.conn.state &= ~context.ConnectionState.CAN_READ
self.conn.state &= ~connection.ConnectionState.CAN_READ
if self.tunnel_state is TunnelState.OPEN:
yield from self.receive_close()
elif self.tunnel_state is TunnelState.ESTABLISHING:
@ -107,9 +108,9 @@ class TunnelLayer(layer.Layer):
elif isinstance(command, commands.CloseConnection):
if self.conn != self.tunnel_connection:
if command.half_close:
self.conn.state &= ~context.ConnectionState.CAN_WRITE
self.conn.state &= ~connection.ConnectionState.CAN_WRITE
else:
self.conn.state = context.ConnectionState.CLOSED
self.conn.state = connection.ConnectionState.CLOSED
yield from self.send_close(command.half_close)
elif isinstance(command, commands.OpenConnection):
# create our own OpenConnection command object that blocks here.

26
mitmproxy/test/tflow.py
View File

@ -1,12 +1,12 @@
import uuid
from mitmproxy import connection
from mitmproxy import controller
from mitmproxy import flow
from mitmproxy import http
from mitmproxy import tcp
from mitmproxy import websocket
from mitmproxy.net import http as net_http
from mitmproxy.proxy import context
from mitmproxy.net.http import status_codes
from mitmproxy.test import tutils
from wsproto.frame_protocol import Opcode
@ -37,7 +37,7 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None,
if server_conn is True:
server_conn = tserver_conn()
if handshake_flow is True:
req = http.HTTPRequest(
req = http.Request(
"example.com",
80,
b"GET",
@ -45,7 +45,7 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None,
b"example.com",
b"/ws",
b"HTTP/1.1",
headers=net_http.Headers(
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
@ -57,11 +57,11 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None,
timestamp_end=946681201,
)
resp = http.HTTPResponse(
resp = http.Response(
b"HTTP/1.1",
101,
reason=net_http.status_codes.RESPONSES.get(101),
headers=net_http.Headers(
reason=status_codes.RESPONSES.get(101),
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
@ -99,8 +99,8 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None):
"""
@type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection
@type server_conn: bool | None | mitmproxy.proxy.connection.ServerConnection
@type req: bool | None | mitmproxy.proxy.protocol.http.HTTPRequest
@type resp: bool | None | mitmproxy.proxy.protocol.http.HTTPResponse
@type req: bool | None | mitmproxy.proxy.protocol.http.Request
@type resp: bool | None | mitmproxy.proxy.protocol.http.Response
@type err: bool | None | mitmproxy.proxy.protocol.primitives.Error
@return: mitmproxy.proxy.protocol.http.HTTPFlow
"""
@ -144,8 +144,8 @@ def tdummyflow(client_conn=True, server_conn=True, err=None):
return f
def tclient_conn() -> context.Client:
c = context.Client.from_state(dict(
def tclient_conn() -> connection.Client:
c = connection.Client.from_state(dict(
id=str(uuid.uuid4()),
address=("127.0.0.1", 22),
mitmcert=None,
@ -170,8 +170,8 @@ def tclient_conn() -> context.Client:
return c
def tserver_conn() -> context.Server:
c = context.Server.from_state(dict(
def tserver_conn() -> connection.Server:
c = connection.Server.from_state(dict(
id=str(uuid.uuid4()),
address=("address", 22),
source_address=("address", 22),

2
mitmproxy/test/tutils.py
View File

@ -1,4 +1,4 @@
from mitmproxy.net import http
from mitmproxy import http
def treq(**kwargs) -> http.Request:

2
mitmproxy/tools/console/consoleaddons.py
View File

@ -419,7 +419,7 @@ class ConsoleAddon:
flow.response is None
)
if require_dummy_response:
flow.response = http.HTTPResponse.make()
flow.response = http.Response.make()
if flow_part == "cookies":
self.master.switch_view("edit_focus_cookies")
elif flow_part == "urlencoded form":

4
mitmproxy/tools/console/flowdetailview.py
View File

@ -24,8 +24,8 @@ def flowdetails(state, flow: mitmproxy.flow.Flow):
sc = flow.server_conn
cc = flow.client_conn
req: typing.Optional[http.HTTPRequest]
resp: typing.Optional[http.HTTPResponse]
req: typing.Optional[http.Request]
resp: typing.Optional[http.Response]
if isinstance(flow, http.HTTPFlow):
req = flow.request
resp = flow.response

2
mitmproxy/tools/console/flowview.py
View File

@ -205,7 +205,7 @@ class FlowDetails(tabs.Tabs):
if error:
self.master.log.debug(error)
# Give hint that you have to tab for the response.
if description == "No content" and isinstance(message, http.HTTPRequest):
if description == "No content" and isinstance(message, http.Request):
description = "No request content"
# If the users has a wide terminal, he gets fewer lines; this should not be an issue.

2
mitmproxy/tools/console/grideditor/editors.py
View File

@ -2,7 +2,7 @@ import urwid
import typing
from mitmproxy import exceptions
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
from mitmproxy.tools.console import layoutwidget
from mitmproxy.tools.console import signals
from mitmproxy.tools.console.grideditor import base

1
setup.cfg
View File

@ -57,7 +57,6 @@ exclude =
mitmproxy/master.py
mitmproxy/net/check.py
mitmproxy/net/http/cookies.py
mitmproxy/net/http/headers.py
mitmproxy/net/http/message.py
mitmproxy/net/http/multipart.py
mitmproxy/net/tcp.py

2
test/examples/test_examples.py
View File

@ -2,7 +2,7 @@ from mitmproxy import contentviews
from mitmproxy.test import tflow
from mitmproxy.test import tutils
from mitmproxy.test import taddons
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
from ..mitmproxy import tservers

5
test/mitmproxy/addons/test_block.py
View File

@ -1,7 +1,7 @@
import pytest
from mitmproxy import connection
from mitmproxy.addons import block
from mitmproxy.proxy import context
from mitmproxy.test import taddons
@ -28,7 +28,6 @@ from mitmproxy.test import taddons
(True, False, True, ("2001:4860:4860::8888",)),
(True, False, True, (r"2001:4860:4860::8888%scope",)),
# block_private: loopback
(False, True, False, ("127.0.0.1",)),
(False, True, False, ("::1",)),
@ -56,6 +55,6 @@ async def test_block_global(block_global, block_private, should_be_killed, addre
ar = block.Block()
with taddons.context(ar) as tctx:
tctx.configure(ar, block_global=block_global, block_private=block_private)
client = context.Client(address, ("127.0.0.1", 8080), 1607699500)
client = connection.Client(address, ("127.0.0.1", 8080), 1607699500)
ar.client_connected(client)
assert bool(client.error) == should_be_killed

2
test/mitmproxy/addons/test_clientplayback.py
View File

@ -5,7 +5,7 @@ import pytest
from mitmproxy.addons.clientplayback import ClientPlayback, ReplayHandler
from mitmproxy.exceptions import CommandError, OptionsError
from mitmproxy.proxy.context import Address
from mitmproxy.connection import Address
from mitmproxy.test import taddons, tflow

2
test/mitmproxy/addons/test_dumper.py
View File

@ -6,7 +6,7 @@ import pytest
from mitmproxy import exceptions
from mitmproxy.addons import dumper
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
from mitmproxy.test import taddons
from mitmproxy.test import tflow
from mitmproxy.test import tutils

3
test/mitmproxy/addons/test_next_layer.py
View File

@ -2,6 +2,7 @@ from unittest.mock import MagicMock
import pytest
from mitmproxy import connection
from mitmproxy.addons.next_layer import NextLayer
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy import context, layers
@ -10,7 +11,7 @@ from mitmproxy.test import taddons
@pytest.fixture
def tctx():
context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
client_hello_no_extensions = bytes.fromhex(

2
test/mitmproxy/addons/test_proxyserver.py
View File

@ -6,7 +6,7 @@ import pytest
from mitmproxy.addons.proxyserver import Proxyserver
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy import layers
from mitmproxy.proxy.context import Address
from mitmproxy.connection import Address
from mitmproxy.test import taddons

18
test/mitmproxy/addons/test_tlsconfig.py
View File

@ -6,7 +6,7 @@ from typing import Union
import pytest
from OpenSSL import SSL
from mitmproxy import certs
from mitmproxy import certs, connection
from mitmproxy.addons import tlsconfig
from mitmproxy.proxy import context
from mitmproxy.proxy.layers import tls
@ -54,7 +54,7 @@ class TestTlsConfig:
with taddons.context(ta) as tctx:
ta.configure(["confdir"])
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
# Edge case first: We don't have _any_ idea about the server, so we just return "mitmproxy" as subject.
entry = ta.get_cert(ctx)
@ -77,7 +77,7 @@ class TestTlsConfig:
# only really testing for coverage here, there's no point in mirroring the individual conditions
ta = tlsconfig.TlsConfig()
with taddons.context(ta) as tctx:
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ch = tls.ClientHelloData(ctx)
ta.tls_clienthello(ch)
assert not ch.establish_server_tls_first
@ -113,7 +113,7 @@ class TestTlsConfig:
certs=[tdata.path("mitmproxy/net/data/verificationcerts/trusted-leaf.pem")],
ciphers_client="ECDHE-ECDSA-AES128-GCM-SHA256",
)
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
tls_start = tls.TlsStartData(ctx.client, context=ctx)
ta.tls_start(tls_start)
@ -125,7 +125,7 @@ class TestTlsConfig:
def test_create_proxy_server_ssl_conn_verify_failed(self):
ta = tlsconfig.TlsConfig()
with taddons.context(ta) as tctx:
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx.client.alpn_offers = [b"h2"]
ctx.client.cipher_list = ["TLS_AES_256_GCM_SHA384", "ECDHE-RSA-AES128-SHA"]
ctx.server.address = ("example.mitmproxy.org", 443)
@ -140,7 +140,7 @@ class TestTlsConfig:
def test_create_proxy_server_ssl_conn_verify_ok(self, tdata):
ta = tlsconfig.TlsConfig()
with taddons.context(ta) as tctx:
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx.server.address = ("example.mitmproxy.org", 443)
tctx.configure(ta, ssl_verify_upstream_trusted_ca=tdata.path(
"mitmproxy/net/data/verificationcerts/trusted-root.crt"))
@ -154,7 +154,7 @@ class TestTlsConfig:
def test_create_proxy_server_ssl_conn_insecure(self):
ta = tlsconfig.TlsConfig()
with taddons.context(ta) as tctx:
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx.server.address = ("example.mitmproxy.org", 443)
tctx.configure(
@ -173,7 +173,7 @@ class TestTlsConfig:
def test_alpn_selection(self):
ta = tlsconfig.TlsConfig()
with taddons.context(ta) as tctx:
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx.server.address = ("example.mitmproxy.org", 443)
tls_start = tls.TlsStartData(ctx.server, context=ctx)
@ -203,7 +203,7 @@ class TestTlsConfig:
def test_client_cert_file(self, tdata, client_certs):
ta = tlsconfig.TlsConfig()
with taddons.context(ta) as tctx:
ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
ctx.server.address = ("example.mitmproxy.org", 443)
tctx.configure(
ta,

2
test/mitmproxy/net/http/http1/test_assemble.py
View File

@ -1,6 +1,6 @@
import pytest
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
from mitmproxy.net.http.http1.assemble import (
assemble_request, assemble_request_head, assemble_response,
assemble_response_head, _assemble_request_line, _assemble_request_headers,

2
test/mitmproxy/net/http/http1/test_read.py
View File

@ -1,6 +1,6 @@
import pytest
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
from mitmproxy.net.http.http1.read import (
read_request_head,
read_response_head, connection_close, expected_http_body_size,

68
test/mitmproxy/net/http/test_headers.py
View File

@ -1,69 +1,6 @@
import collections
import pytest
from mitmproxy.net.http.headers import Headers, parse_content_type, assemble_content_type
class TestHeaders:
def _2host(self):
return Headers(
(
(b"Host", b"example.com"),
(b"host", b"example.org")
)
)
def test_init(self):
headers = Headers()
assert len(headers) == 0
headers = Headers([[b"Host", b"example.com"]])
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(Host="example.com")
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"]],
Host="example.com"
)
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"], [b"Accept", b"text/plain"]],
Host="example.com"
)
assert len(headers) == 2
assert headers["Host"] == "example.com"
assert headers["Accept"] == "text/plain"
with pytest.raises(TypeError):
Headers([[b"Host", "not-bytes"]])
def test_set(self):
headers = Headers()
headers["foo"] = "1"
headers[b"bar"] = b"2"
headers["baz"] = b"3"
with pytest.raises(TypeError):
headers["foobar"] = 42
assert len(headers) == 3
def test_bytes(self):
headers = Headers(Host="example.com")
assert bytes(headers) == b"Host: example.com\r\n"
headers = Headers([
[b"Host", b"example.com"],
[b"Accept", b"text/plain"]
])
assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n"
headers = Headers()
assert bytes(headers) == b""
from mitmproxy.net.http.headers import parse_content_type, assemble_content_type
def test_parse_content_type():
@ -79,4 +16,5 @@ def test_assemble_content_type():
p = assemble_content_type
assert p("text", "html", {}) == "text/html"
assert p("text", "html", {"charset": "utf8"}) == "text/html; charset=utf8"
assert p("text", "html", collections.OrderedDict([("charset", "utf8"), ("foo", "bar")])) == "text/html; charset=utf8; foo=bar"
assert p("text", "html",
collections.OrderedDict([("charset", "utf8"), ("foo", "bar")])) == "text/html; charset=utf8; foo=bar"

314
test/mitmproxy/net/http/test_message.py
View File

@ -1,314 +0,0 @@
import pytest
from mitmproxy.test import tutils
from mitmproxy.net import http
def _test_passthrough_attr(message, attr):
assert getattr(message, attr) == getattr(message.data, attr)
setattr(message, attr, b"foo")
assert getattr(message.data, attr) == b"foo"
def _test_decoded_attr(message, attr):
assert getattr(message, attr) == getattr(message.data, attr).decode("utf8")
# Set str, get raw bytes
setattr(message, attr, "foo")
assert getattr(message.data, attr) == b"foo"
# Set raw bytes, get decoded
setattr(message.data, attr, b"BAR") # use uppercase so that we can also cover request.method
assert getattr(message, attr) == "BAR"
# Set bytes, get raw bytes
setattr(message, attr, b"baz")
assert getattr(message.data, attr) == b"baz"
# Set UTF8
setattr(message, attr, "Non-Autorisé")
assert getattr(message.data, attr) == b"Non-Autoris\xc3\xa9"
# Don't fail on garbage
setattr(message.data, attr, b"FOO\xBF\x00BAR")
assert getattr(message, attr).startswith("FOO")
assert getattr(message, attr).endswith("BAR")
# foo.bar = foo.bar should not cause any side effects.
d = getattr(message, attr)
setattr(message, attr, d)
assert getattr(message.data, attr) == b"FOO\xBF\x00BAR"
class TestMessageData:
def test_eq(self):
data = tutils.tresp(timestamp_start=42, timestamp_end=42).data
same = tutils.tresp(timestamp_start=42, timestamp_end=42).data
assert data == same
other = tutils.tresp(content=b"foo").data
assert data != other
assert data != 0
def test_serializable(self):
data1 = tutils.tresp(timestamp_start=42, timestamp_end=42).data
data2 = tutils.tresp().data.from_state(data1.get_state()) # ResponseData.from_state()
assert data1 == data2
class TestMessage:
def test_init(self):
resp = tutils.tresp()
assert resp.data
def test_eq_ne(self):
resp = tutils.tresp(timestamp_start=42, timestamp_end=42)
same = tutils.tresp(timestamp_start=42, timestamp_end=42)
assert resp.data == same.data
other = tutils.tresp(timestamp_start=0, timestamp_end=0)
assert resp.data != other.data
assert resp != 0
def test_serializable(self):
resp = tutils.tresp()
resp2 = http.Response.from_state(resp.get_state())
assert resp.data == resp2.data
def test_content_length_update(self):
resp = tutils.tresp()
resp.content = b"foo"
assert resp.data.content == b"foo"
assert resp.headers["content-length"] == "3"
resp.content = b""
assert resp.data.content == b""
assert resp.headers["content-length"] == "0"
resp.raw_content = b"bar"
assert resp.data.content == b"bar"
assert resp.headers["content-length"] == "0"
def test_headers(self):
_test_passthrough_attr(tutils.tresp(), "headers")
def test_timestamp_start(self):
_test_passthrough_attr(tutils.tresp(), "timestamp_start")
def test_timestamp_end(self):
_test_passthrough_attr(tutils.tresp(), "timestamp_end")
def test_http_version(self):
_test_decoded_attr(tutils.tresp(), "http_version")
assert tutils.tresp(http_version=b"HTTP/1.0").is_http10
assert tutils.tresp(http_version=b"HTTP/1.1").is_http11
assert tutils.tresp(http_version=b"HTTP/2.0").is_http2
class TestMessageContentEncoding:
def test_simple(self):
r = tutils.tresp()
assert r.raw_content == b"message"
assert "content-encoding" not in r.headers
r.encode("gzip")
assert r.headers["content-encoding"]
assert r.raw_content != b"message"
assert r.content == b"message"
assert r.raw_content != b"message"
def test_update_content_length_header(self):
r = tutils.tresp()
assert int(r.headers["content-length"]) == 7
r.encode("gzip")
assert int(r.headers["content-length"]) == 27
r.decode()
assert int(r.headers["content-length"]) == 7
def test_modify(self):
r = tutils.tresp()
assert "content-encoding" not in r.headers
r.encode("gzip")
r.content = b"foo"
assert r.raw_content != b"foo"
r.decode()
assert r.raw_content == b"foo"
with pytest.raises(TypeError):
r.content = "foo"
def test_unknown_ce(self):
r = tutils.tresp()
r.headers["content-encoding"] = "zopfli"
r.raw_content = b"foo"
with pytest.raises(ValueError):
assert r.content
assert r.headers["content-encoding"]
assert r.get_content(strict=False) == b"foo"
def test_utf8_as_ce(self):
r = tutils.tresp()
r.headers["content-encoding"] = "utf8"
r.raw_content = b"foo"
with pytest.raises(ValueError):
assert r.content
assert r.headers["content-encoding"]
assert r.get_content(strict=False) == b"foo"
def test_cannot_decode(self):
r = tutils.tresp()
r.encode("gzip")
r.raw_content = b"foo"
with pytest.raises(ValueError):
assert r.content
assert r.headers["content-encoding"]
assert r.get_content(strict=False) == b"foo"
with pytest.raises(ValueError):
r.decode()
assert r.raw_content == b"foo"
assert "content-encoding" in r.headers
r.decode(strict=False)
assert r.content == b"foo"
assert "content-encoding" not in r.headers
def test_none(self):
r = tutils.tresp(content=None)
assert r.content is None
r.content = b"foo"
assert r.content is not None
r.content = None
assert r.content is None
def test_cannot_encode(self):
r = tutils.tresp()
r.encode("gzip")
r.content = None
assert r.headers["content-encoding"]
assert r.raw_content is None
r.headers["content-encoding"] = "zopfli"
r.content = b"foo"
assert "content-encoding" not in r.headers
assert r.raw_content == b"foo"
with pytest.raises(ValueError):
r.encode("zopfli")
assert r.raw_content == b"foo"
assert "content-encoding" not in r.headers
class TestMessageText:
def test_simple(self):
r = tutils.tresp(content=b'\xfc')
assert r.raw_content == b"\xfc"
assert r.content == b"\xfc"
assert r.text == "ü"
r.encode("gzip")
assert r.text == "ü"
r.decode()
assert r.text == "ü"
r.headers["content-type"] = "text/html; charset=latin1"
r.content = b"\xc3\xbc"
assert r.text == "ü"
r.headers["content-type"] = "text/html; charset=utf8"
assert r.text == "ü"
def test_guess_json(self):
r = tutils.tresp(content=b'"\xc3\xbc"')
r.headers["content-type"] = "application/json"
assert r.text == '"ü"'
def test_guess_meta_charset(self):
r = tutils.tresp(content=b'<meta http-equiv="content-type" '
b'content="text/html;charset=gb2312">\xe6\x98\x8e\xe4\xbc\xaf')
# "鏄庝集" is decoded form of \xe6\x98\x8e\xe4\xbc\xaf in gb18030
assert "鏄庝集" in r.text
def test_guess_css_charset(self):
# @charset but not text/css
r = tutils.tresp(content=b'@charset "gb2312";'
b'#foo::before {content: "\xe6\x98\x8e\xe4\xbc\xaf"}')
# "鏄庝集" is decoded form of \xe6\x98\x8e\xe4\xbc\xaf in gb18030
assert "鏄庝集" not in r.text
# @charset not at the beginning
r = tutils.tresp(content=b'foo@charset "gb2312";'
b'#foo::before {content: "\xe6\x98\x8e\xe4\xbc\xaf"}')
r.headers["content-type"] = "text/css"
# "鏄庝集" is decoded form of \xe6\x98\x8e\xe4\xbc\xaf in gb18030
assert "鏄庝集" not in r.text
# @charset and text/css
r = tutils.tresp(content=b'@charset "gb2312";'
b'#foo::before {content: "\xe6\x98\x8e\xe4\xbc\xaf"}')
r.headers["content-type"] = "text/css"
# "鏄庝集" is decoded form of \xe6\x98\x8e\xe4\xbc\xaf in gb18030
assert "鏄庝集" in r.text
def test_guess_latin_1(self):
r = tutils.tresp(content=b"\xF0\xE2")
assert r.text == "ðâ"
def test_none(self):
r = tutils.tresp(content=None)
assert r.text is None
r.text = "foo"
assert r.text is not None
r.text = None
assert r.text is None
def test_modify(self):
r = tutils.tresp()
r.text = "ü"
assert r.raw_content == b"\xfc"
r.headers["content-type"] = "text/html; charset=utf8"
r.text = "ü"
assert r.raw_content == b"\xc3\xbc"
assert r.headers["content-length"] == "2"
def test_unknown_ce(self):
r = tutils.tresp()
r.headers["content-type"] = "text/html; charset=wtf"
r.raw_content = b"foo"
with pytest.raises(ValueError):
assert r.text == "foo"
assert r.get_text(strict=False) == "foo"
def test_cannot_decode(self):
r = tutils.tresp()
r.headers["content-type"] = "text/html; charset=utf8"
r.raw_content = b"\xFF"
with pytest.raises(ValueError):
assert r.text
assert r.get_text(strict=False) == '\udcff'
def test_cannot_encode(self):
r = tutils.tresp()
r.content = None
assert "content-type" not in r.headers
assert r.raw_content is None
r.headers["content-type"] = "text/html; charset=latin1; foo=bar"
r.text = "☃"
assert r.headers["content-type"] == "text/html; charset=utf-8; foo=bar"
assert r.raw_content == b'\xe2\x98\x83'
r.headers["content-type"] = "gibberish"
r.text = "☃"
assert r.headers["content-type"] == "text/plain; charset=utf-8"
assert r.raw_content == b'\xe2\x98\x83'
del r.headers["content-type"]
r.text = "☃"
assert r.headers["content-type"] == "text/plain; charset=utf-8"
assert r.raw_content == b'\xe2\x98\x83'
r.headers["content-type"] = "text/html; charset=latin1"
r.text = '\udcff'
assert r.headers["content-type"] == "text/html; charset=utf-8"
assert r.raw_content == b"\xFF"

2
test/mitmproxy/net/http/test_multipart.py
View File

@ -1,6 +1,6 @@
import pytest
from mitmproxy.net.http import Headers
from mitmproxy.http import Headers
from mitmproxy.net.http import multipart

376
test/mitmproxy/net/http/test_request.py
View File

@ -1,376 +0,0 @@
from unittest import mock
import pytest
from mitmproxy.net.http import Headers, Request
from mitmproxy.test.tutils import treq
from .test_message import _test_decoded_attr, _test_passthrough_attr
class TestRequestData:
def test_init(self):
with pytest.raises(UnicodeEncodeError):
treq(method="fööbär")
with pytest.raises(UnicodeEncodeError):
treq(scheme="fööbär")
assert treq(host="fööbär").host == "fööbär"
with pytest.raises(UnicodeEncodeError):
treq(path="/fööbär")
with pytest.raises(UnicodeEncodeError):
treq(http_version="föö/bä.r")
with pytest.raises(ValueError):
treq(headers="foobar")
with pytest.raises(ValueError):
treq(content="foobar")
with pytest.raises(ValueError):
treq(trailers="foobar")
assert isinstance(treq(headers=()).headers, Headers)
assert isinstance(treq(trailers=()).trailers, Headers)
class TestRequestCore:
"""
Tests for addons and the attributes that are directly proxied from the data structure
"""
def test_repr(self):
request = treq()
assert repr(request) == "Request(GET address:22/path)"
request.host = None
assert repr(request) == "Request(GET /path)"
def test_init_conv(self):
assert Request(
b"example.com",
80,
"GET",
"http",
"example.com",
"/",
"HTTP/1.1",
(),
None,
(),
0,
0,
) # type: ignore
def test_make(self):
r = Request.make("GET", "https://example.com/")
assert r.method == "GET"
assert r.scheme == "https"
assert r.host == "example.com"
assert r.port == 443
assert r.path == "/"
r = Request.make("GET", "https://example.com/", "content", {"Foo": "bar"})
assert r.content == b"content"
assert r.headers["content-length"] == "7"
assert r.headers["Foo"] == "bar"
Request.make("GET", "https://example.com/", content=b"content")
with pytest.raises(TypeError):
Request.make("GET", "https://example.com/", content=42)
r = Request.make("GET", "https://example.com/", headers=[(b"foo", b"bar")])
assert r.headers["foo"] == "bar"
r = Request.make("GET", "https://example.com/", headers=({"foo": "baz"}))
assert r.headers["foo"] == "baz"
r = Request.make("GET", "https://example.com/", headers=Headers(foo="qux"))
assert r.headers["foo"] == "qux"
with pytest.raises(TypeError):
Request.make("GET", "https://example.com/", headers=42)
def test_first_line_format(self):
assert treq(method=b"CONNECT").first_line_format == "authority"
assert treq(authority=b"example.com").first_line_format == "absolute"
assert treq(authority=b"").first_line_format == "relative"
def test_method(self):
_test_decoded_attr(treq(), "method")
def test_scheme(self):
_test_decoded_attr(treq(), "scheme")
def test_port(self):
_test_passthrough_attr(treq(), "port")
def test_path(self):
_test_decoded_attr(treq(), "path")
def test_authority(self):
request = treq()
assert request.authority == request.data.authority.decode("idna")
# Test IDNA encoding
# Set str, get raw bytes
request.authority = "ídna.example"
assert request.data.authority == b"xn--dna-qma.example"
# Set raw bytes, get decoded
request.data.authority = b"xn--idn-gla.example"
assert request.authority == "idná.example"
# Set bytes, get raw bytes
request.authority = b"xn--dn-qia9b.example"
assert request.data.authority == b"xn--dn-qia9b.example"
# IDNA encoding is not bijective
request.authority = "fußball"
assert request.authority == "fussball"
# Don't fail on garbage
request.data.authority = b"foo\xFF\x00bar"
assert request.authority.startswith("foo")
assert request.authority.endswith("bar")
# foo.bar = foo.bar should not cause any side effects.
d = request.authority
request.authority = d
assert request.data.authority == b"foo\xFF\x00bar"
def test_host_update_also_updates_header(self):
request = treq()
assert "host" not in request.headers
request.host = "example.com"
assert "host" not in request.headers
request.headers["Host"] = "foo"
request.authority = "foo"
request.host = "example.org"
assert request.headers["Host"] == "example.org"
assert request.authority == "example.org:22"
def test_get_host_header(self):
no_hdr = treq()
assert no_hdr.host_header is None
h1 = treq(
headers=((b"host", b"header.example.com"),),
authority=b"authority.example.com"
)
assert h1.host_header == "header.example.com"
h2 = h1.copy()
h2.http_version = "HTTP/2.0"
assert h2.host_header == "authority.example.com"
h2_host_only = h2.copy()
h2_host_only.authority = ""
assert h2_host_only.host_header == "header.example.com"
def test_modify_host_header(self):
h1 = treq()
assert "host" not in h1.headers
h1.host_header = "example.com"
assert h1.headers["Host"] == "example.com"
assert not h1.authority
h1.host_header = None
assert "host" not in h1.headers
assert not h1.authority
h2 = treq(http_version=b"HTTP/2.0")
h2.host_header = "example.org"
assert "host" not in h2.headers
assert h2.authority == "example.org"
h2.headers["Host"] = "example.org"
h2.host_header = "foo.example.com"
assert h2.headers["Host"] == "foo.example.com"
assert h2.authority == "foo.example.com"
h2.host_header = None
assert "host" not in h2.headers
assert not h2.authority
class TestRequestUtils:
"""
Tests for additional convenience methods.
"""
def test_url(self):
request = treq()
assert request.url == "http://address:22/path"
request.url = "https://otheraddress:42/foo"
assert request.scheme == "https"
assert request.host == "otheraddress"
assert request.port == 42
assert request.path == "/foo"
with pytest.raises(ValueError):
request.url = "not-a-url"
def test_url_options(self):
request = treq(method=b"OPTIONS", path=b"*")
assert request.url == "http://address:22"
def test_url_authority(self):
request = treq(method=b"CONNECT")
assert request.url == "address:22"
def test_pretty_host(self):
request = treq()
# Without host header
assert request.pretty_host == "address"
assert request.host == "address"
# Same port as self.port (22)
request.headers["host"] = "other:22"
assert request.pretty_host == "other"
# Invalid IDNA
request.headers["host"] = ".disqus.com"
assert request.pretty_host == ".disqus.com"
def test_pretty_url(self):
request = treq()
# Without host header
assert request.url == "http://address:22/path"
assert request.pretty_url == "http://address:22/path"
request.headers["host"] = "other:22"
assert request.pretty_url == "http://other:22/path"
request = treq(method=b"CONNECT", authority=b"example:44")
assert request.pretty_url == "example:44"
def test_pretty_url_options(self):
request = treq(method=b"OPTIONS", path=b"*")
assert request.pretty_url == "http://address:22"
def test_pretty_url_authority(self):
request = treq(method=b"CONNECT", authority="address:22")
assert request.pretty_url == "address:22"
def test_get_query(self):
request = treq()
assert not request.query
request.url = "http://localhost:80/foo?bar=42"
assert dict(request.query) == {"bar": "42"}
def test_set_query(self):
request = treq()
assert not request.query
request.query["foo"] = "bar"
assert request.query["foo"] == "bar"
assert request.path == "/path?foo=bar"
request.query = [('foo', 'bar')]
assert request.query["foo"] == "bar"
assert request.path == "/path?foo=bar"
def test_get_cookies_none(self):
request = treq()
request.headers = Headers()
assert not request.cookies
def test_get_cookies_single(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
assert len(request.cookies) == 1
assert request.cookies['cookiename'] == 'cookievalue'
def test_get_cookies_double(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'cookievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_get_cookies_withequalsign(self):
request = treq()
request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'coo=kievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_set_cookies(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
result = request.cookies
result["cookiename"] = "foo"
assert request.cookies["cookiename"] == "foo"
request.cookies = [["one", "uno"], ["two", "due"]]
assert request.cookies["one"] == "uno"
assert request.cookies["two"] == "due"
def test_get_path_components(self):
request = treq(path=b"/foo/bar")
assert request.path_components == ("foo", "bar")
def test_set_path_components(self):
request = treq()
request.path_components = ["foo", "baz"]
assert request.path == "/foo/baz"
request.path_components = []
assert request.path == "/"
request.path_components = ["foo", "baz"]
request.query["hello"] = "hello"
assert request.path_components == ("foo", "baz")
request.path_components = ["abc"]
assert request.path == "/abc?hello=hello"
def test_anticache(self):
request = treq()
request.headers["If-Modified-Since"] = "foo"
request.headers["If-None-Match"] = "bar"
request.anticache()
assert "If-Modified-Since" not in request.headers
assert "If-None-Match" not in request.headers
def test_anticomp(self):
request = treq()
request.headers["Accept-Encoding"] = "foobar"
request.anticomp()
assert request.headers["Accept-Encoding"] == "identity"
def test_constrain_encoding(self):
request = treq()
h = request.headers.copy()
request.constrain_encoding() # no-op if there is no accept_encoding header.
assert request.headers == h
request.headers["Accept-Encoding"] = "identity, gzip, foo"
request.constrain_encoding()
assert "foo" not in request.headers["Accept-Encoding"]
assert "gzip" in request.headers["Accept-Encoding"]
def test_get_urlencoded_form(self):
request = treq(content=b"foobar=baz")
assert not request.urlencoded_form
request.headers["Content-Type"] = "application/x-www-form-urlencoded"
assert list(request.urlencoded_form.items()) == [("foobar", "baz")]
request.raw_content = b"\xFF"
assert len(request.urlencoded_form) == 1
def test_set_urlencoded_form(self):
request = treq(content=b"\xec\xed")
request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')]
assert request.headers["Content-Type"] == "application/x-www-form-urlencoded"
assert request.content
def test_get_multipart_form(self):
request = treq(content=b"foobar")
assert not request.multipart_form
request.headers["Content-Type"] = "multipart/form-data"
assert list(request.multipart_form.items()) == []
with mock.patch('mitmproxy.net.http.multipart.decode') as m:
m.side_effect = ValueError
assert list(request.multipart_form.items()) == []
def test_set_multipart_form(self):
request = treq()
request.multipart_form = [("file", "shell.jpg"), ("file_size", "1000")]
assert request.headers["Content-Type"] == 'multipart/form-data'
assert request.content is None

173
test/mitmproxy/net/http/test_response.py
View File

@ -1,173 +0,0 @@
import email
import time
import pytest
from unittest import mock
from mitmproxy.net.http import Headers
from mitmproxy.net.http import Response
from mitmproxy.net.http.cookies import CookieAttrs
from mitmproxy.test.tutils import tresp
from .test_message import _test_passthrough_attr
class TestResponseData:
def test_init(self):
with pytest.raises(ValueError):
tresp(headers="foobar")
with pytest.raises(UnicodeEncodeError):
tresp(http_version="föö/bä.r")
with pytest.raises(UnicodeEncodeError):
tresp(reason="fööbär")
with pytest.raises(ValueError):
tresp(content="foobar")
with pytest.raises(ValueError):
tresp(trailers="foobar")
assert isinstance(tresp(headers=()).headers, Headers)
assert isinstance(tresp(trailers=()).trailers, Headers)
class TestResponseCore:
"""
Tests for addons and the attributes that are directly proxied from the data structure
"""
def test_repr(self):
response = tresp()
assert repr(response) == "Response(200, unknown content type, 7b)"
response.content = None
assert repr(response) == "Response(200, no content)"
def test_make(self):
r = Response.make()
assert r.status_code == 200
assert r.content == b""
r = Response.make(418, "teatime")
assert r.status_code == 418
assert r.content == b"teatime"
assert r.headers["content-length"] == "7"
Response.make(content=b"foo")
Response.make(content="foo")
with pytest.raises(TypeError):
Response.make(content=42)
r = Response.make(headers=[(b"foo", b"bar")])
assert r.headers["foo"] == "bar"
r = Response.make(headers=({"foo": "baz"}))
assert r.headers["foo"] == "baz"
r = Response.make(headers=Headers(foo="qux"))
assert r.headers["foo"] == "qux"
with pytest.raises(TypeError):
Response.make(headers=42)
def test_status_code(self):
_test_passthrough_attr(tresp(), "status_code")
def test_reason(self):
resp = tresp()
assert resp.reason == "OK"
resp.reason = "ABC"
assert resp.data.reason == b"ABC"
resp.reason = b"DEF"
assert resp.data.reason == b"DEF"
resp.data.reason = b'cr\xe9e'
assert resp.reason == "crée"
class TestResponseUtils:
"""
Tests for additional convenience methods.
"""
def test_get_cookies_none(self):
resp = tresp()
resp.headers = Headers()
assert not resp.cookies
def test_get_cookies_empty(self):
resp = tresp()
resp.headers = Headers(set_cookie="")
assert not resp.cookies
def test_get_cookies_simple(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=cookievalue")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
def test_get_cookies_with_parameters(self):
resp = tresp()
cookie = "cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly"
resp.headers = Headers(set_cookie=cookie)
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"][0] == "cookievalue"
attrs = result["cookiename"][1]
assert len(attrs) == 4
assert attrs["domain"] == "example.com"
assert attrs["expires"] == "Wed Oct 21 16:29:41 2015"
assert attrs["path"] == "/"
assert attrs["httponly"] == ""
def test_get_cookies_no_value(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=; Expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"][0] == ""
assert len(result["cookiename"][1]) == 2
def test_get_cookies_twocookies(self):
resp = tresp()
resp.headers = Headers([
[b"Set-Cookie", b"cookiename=cookievalue"],
[b"Set-Cookie", b"othercookie=othervalue"]
])
result = resp.cookies
assert len(result) == 2
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
assert "othercookie" in result
assert result["othercookie"] == ("othervalue", CookieAttrs())
def test_set_cookies(self):
resp = tresp()
resp.cookies["foo"] = ("bar", {})
assert len(resp.cookies) == 1
assert resp.cookies["foo"] == ("bar", CookieAttrs())
resp.cookies = [["one", ("uno", CookieAttrs())], ["two", ("due", CookieAttrs())]]
assert list(resp.cookies.keys()) == ["one", "two"]
def test_refresh(self):
r = tresp()
n = time.time()
r.headers["date"] = email.utils.formatdate(n, usegmt=True)
pre = r.headers["date"]
r.refresh(946681202)
assert pre == r.headers["date"]
r.refresh(946681262)
d = email.utils.parsedate_tz(r.headers["date"])
d = email.utils.mktime_tz(d)
# Weird that this is not exact...
assert abs(60 - (d - n)) <= 1
cookie = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"
r.headers["set-cookie"] = cookie
r.refresh()
# Cookie refreshing is tested in test_cookies, we just make sure that it's triggered here.
assert cookie != r.headers["set-cookie"]
with mock.patch('mitmproxy.net.http.cookies.refresh_set_cookie_header') as m:
m.side_effect = ValueError
r.refresh(n)

2
test/mitmproxy/net/http/test_encoding.py → test/mitmproxy/net/test_encoding.py
View File

@ -1,7 +1,7 @@
from unittest import mock
import pytest
from mitmproxy.net.http import encoding
from mitmproxy.net import encoding
@pytest.mark.parametrize("encoder", [

4
test/mitmproxy/proxy/conftest.py
View File

@ -3,7 +3,7 @@ import os
import pytest
from hypothesis import settings
from mitmproxy import options
from mitmproxy import options, connection
from mitmproxy.addons.core import Core
from mitmproxy.addons.proxyserver import Proxyserver
from mitmproxy.addons.termlog import TermLog
@ -17,7 +17,7 @@ def tctx() -> context.Context:
TermLog().load(opts)
Core().load(opts)
return context.Context(
context.Client(
connection.Client(
("client", 1234),
("127.0.0.1", 8080),
1605699329

8
test/mitmproxy/proxy/layers/http/test_http.py
View File

@ -1,12 +1,12 @@
import pytest
from mitmproxy.flow import Error
from mitmproxy.http import HTTPFlow, HTTPResponse
from mitmproxy.http import HTTPFlow, Response
from mitmproxy.net.server_spec import ServerSpec
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy import layer
from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData, Log
from mitmproxy.proxy.context import ConnectionState, Server
from mitmproxy.connection import ConnectionState, Server
from mitmproxy.proxy.events import ConnectionClosed, DataReceived
from mitmproxy.proxy.layers import TCPLayer, http, tls
from mitmproxy.proxy.layers.tcp import TcpStartHook
@ -205,7 +205,7 @@ def test_http_reply_from_proxy(tctx):
"""Test a response served by mitmproxy itself."""
def reply_from_proxy(flow: HTTPFlow):
flow.response = HTTPResponse.make(418)
flow.response = Response.make(418)
assert (
Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False)
@ -843,7 +843,7 @@ def test_kill_flow(tctx, when):
return assert_kill()
if when == "script-response-responseheaders":
assert (playbook
>> reply(side_effect=lambda f: setattr(f, "response", HTTPResponse.make()))
>> reply(side_effect=lambda f: setattr(f, "response", Response.make()))
<< http.HttpResponseHeadersHook(flow))
return assert_kill()
assert (playbook

7
test/mitmproxy/proxy/layers/http/test_http1.py
View File

@ -1,10 +1,11 @@
import pytest
from mitmproxy.net import http
from mitmproxy import http
from mitmproxy.proxy.commands import SendData
from mitmproxy.proxy.events import DataReceived
from mitmproxy.proxy.layers.http import Http1Server, ReceiveHttp, RequestHeaders, RequestEndOfMessage, \
ResponseHeaders, ResponseEndOfMessage, RequestData, Http1Client, ResponseData
from mitmproxy.proxy.layers.http._http1 import make_error_response
from test.mitmproxy.proxy.tutils import Placeholder, Playbook
@ -199,3 +200,7 @@ class TestClient:
>> RequestHeaders(3, req, True)
<< SendData(tctx.server, Placeholder(bytes))
)
def test_make_error_response():
assert make_error_response(543, 'foobar')

7
test/mitmproxy/proxy/layers/http/test_http2.py
View File

@ -7,11 +7,12 @@ import pytest
from h2.errors import ErrorCodes
from mitmproxy.flow import Error
from mitmproxy.http import HTTPFlow
from mitmproxy.net.http import Headers, Request, status_codes
from mitmproxy.http import HTTPFlow, Headers, Request
from mitmproxy.net.http import status_codes
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData
from mitmproxy.proxy.context import Context, Server
from mitmproxy.connection import Server
from mitmproxy.proxy.events import ConnectionClosed, DataReceived
from mitmproxy.proxy.layers import http
from mitmproxy.proxy.layers.http._http2 import split_pseudo_headers, Http2Client

18
test/mitmproxy/proxy/layers/http/test_http_fuzz.py
View File

@ -6,13 +6,13 @@ from hypothesis import example, given
from hypothesis.strategies import binary, booleans, composite, dictionaries, integers, lists, sampled_from, sets, text, \
data
from mitmproxy import options
from mitmproxy import options, connection
from mitmproxy.addons.proxyserver import Proxyserver
from mitmproxy.connection import Server
from mitmproxy.http import HTTPFlow
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy import context, events
from mitmproxy.proxy.commands import OpenConnection, SendData
from mitmproxy.proxy.context import Server
from mitmproxy.proxy.events import DataReceived, Start, ConnectionClosed
from mitmproxy.proxy.layers import http
from test.mitmproxy.proxy.layers.http.hyper_h2_test_helpers import FrameFactory
@ -100,7 +100,7 @@ def h2_responses(draw):
@given(chunks(mutations(h1_requests())))
def test_fuzz_h1_request(data):
tctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
tctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
layer = http.HttpLayer(tctx, HTTPMode.regular)
for _ in layer.handle_event(Start()):
@ -113,8 +113,8 @@ def test_fuzz_h1_request(data):
@given(chunks(mutations(h2_responses())))
@example([b'0 OK\r\n\r\n', b'\r\n', b'5\r\n12345\r\n0\r\n\r\n'])
def test_fuzz_h1_response(data):
tctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
server = Placeholder(context.Server)
tctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
server = Placeholder(connection.Server)
playbook = Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False)
assert (
playbook
@ -207,7 +207,7 @@ def h2_frames(draw):
def h2_layer(opts):
tctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
tctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
tctx.client.alpn = b"h2"
layer = http.HttpLayer(tctx, HTTPMode.regular)
@ -246,9 +246,9 @@ def test_fuzz_h2_request_mutations(chunks):
def _h2_response(chunks):
tctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
tctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
playbook = Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False)
server = Placeholder(context.Server)
server = Placeholder(connection.Server)
assert (
playbook
>> DataReceived(tctx.client, b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n")
@ -314,7 +314,7 @@ def _test_cancel(stream_req, stream_resp, draw):
"""
Test that we don't raise an exception if someone disconnects.
"""
tctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
tctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts)
playbook, cff = start_h2_client(tctx)
flow = Placeholder(HTTPFlow)
server = Placeholder(Server)

3
test/mitmproxy/proxy/layers/http/test_http_version_interop.py
View File

@ -5,9 +5,10 @@ import h2.connection
import h2.events
from mitmproxy.http import HTTPFlow
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData
from mitmproxy.proxy.context import Context, Server
from mitmproxy.connection import Server
from mitmproxy.proxy.events import DataReceived
from mitmproxy.proxy.layers import http
from test.mitmproxy.proxy.layers.http.hyper_h2_test_helpers import FrameFactory

3
test/mitmproxy/proxy/layers/test_modes.py
View File

@ -3,9 +3,10 @@ import copy
import pytest
from mitmproxy import platform
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData, GetSocket, Log
from mitmproxy.proxy.context import Client, Context, Server
from mitmproxy.connection import Client, Server
from mitmproxy.proxy.events import DataReceived, ConnectionClosed
from mitmproxy.proxy.layer import NextLayer, NextLayerHook
from mitmproxy.proxy.layers import http, modes, tcp, tls

2
test/mitmproxy/proxy/layers/test_tcp.py
View File

@ -1,7 +1,7 @@
import pytest
from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData
from mitmproxy.proxy.context import ConnectionState
from mitmproxy.connection import ConnectionState
from mitmproxy.proxy.events import ConnectionClosed, DataReceived
from mitmproxy.proxy.layers import tcp
from mitmproxy.tcp import TCPFlow

9
test/mitmproxy/proxy/layers/test_tls.py
View File

@ -4,8 +4,9 @@ import typing
import pytest
from OpenSSL import SSL
from mitmproxy import connection
from mitmproxy.connection import ConnectionState, Server
from mitmproxy.proxy import commands, context, events, layer
from mitmproxy.proxy.context import ConnectionState
from mitmproxy.proxy.layers import tls
from mitmproxy.utils import data
from test.mitmproxy.proxy import tutils
@ -121,7 +122,7 @@ class SSLTest:
return self.obj.do_handshake()
def _test_echo(playbook: tutils.Playbook, tssl: SSLTest, conn: context.Connection) -> None:
def _test_echo(playbook: tutils.Playbook, tssl: SSLTest, conn: connection.Connection) -> None:
tssl.obj.write(b"Hello World")
data = tutils.Placeholder(bytes)
assert (
@ -145,7 +146,7 @@ class TlsEchoLayer(tutils.EchoLayer):
yield from super()._handle_event(event)
def interact(playbook: tutils.Playbook, conn: context.Connection, tssl: SSLTest):
def interact(playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest):
data = tutils.Placeholder(bytes)
assert (
playbook
@ -383,7 +384,7 @@ class TestClientTLS:
# Echo
_test_echo(playbook, tssl_client, tctx.client)
other_server = context.Server(None)
other_server = Server(None)
assert (
playbook
>> events.DataReceived(other_server, b"Plaintext")

5
test/mitmproxy/proxy/layers/test_websocket.py
View File

@ -5,11 +5,10 @@ import pytest
import wsproto
import wsproto.events
from mitmproxy.http import HTTPFlow
from mitmproxy.net.http import Request, Response
from mitmproxy.http import HTTPFlow, Request, Response
from mitmproxy.proxy.layers.http import HTTPMode
from mitmproxy.proxy.commands import SendData, CloseConnection, Log
from mitmproxy.proxy.context import ConnectionState
from mitmproxy.connection import ConnectionState
from mitmproxy.proxy.events import DataReceived, ConnectionClosed
from mitmproxy.proxy.layers import http, websocket
from mitmproxy.websocket import WebSocketFlow

7
test/mitmproxy/proxy/test_commands.py
View File

@ -2,13 +2,14 @@ from dataclasses import dataclass
import pytest
from mitmproxy import connection
from mitmproxy.hooks import all_hooks
from mitmproxy.proxy import commands, context
from mitmproxy.proxy import commands
@pytest.fixture
def tconn() -> context.Server:
return context.Server(None)
def tconn() -> connection.Server:
return connection.Server(None)
def test_dataclasses(tconn):

76
test/mitmproxy/proxy/test_context.py
View File

@ -2,82 +2,6 @@ from mitmproxy.proxy import context
from mitmproxy.test import tflow, taddons
class TestConnection:
def test_basic(self):
c = context.Client(
("127.0.0.1", 52314),
("127.0.0.1", 8080),
1607780791
)
assert not c.tls_established
c.timestamp_tls_setup = 1607780792
assert c.tls_established
assert c.connected
c.state = context.ConnectionState.CAN_WRITE
assert not c.connected
def test_eq(self):
c = tflow.tclient_conn()
c2 = c.copy()
assert c == c
assert c != c2
assert c != 42
assert hash(c) != hash(c2)
c2.id = c.id
assert c == c2
class TestClient:
def test_basic(self):
c = context.Client(
("127.0.0.1", 52314),
("127.0.0.1", 8080),
1607780791
)
assert repr(c)
assert str(c)
c.timestamp_tls_setup = 1607780791
assert str(c)
c.alpn = b"foo"
assert str(c) == "Client(127.0.0.1:52314, state=open, alpn=foo)"
def test_state(self):
c = tflow.tclient_conn()
assert context.Client.from_state(c.get_state()).get_state() == c.get_state()
c2 = tflow.tclient_conn()
assert c != c2
c2.timestamp_start = 42
c.set_state(c2.get_state())
assert c.timestamp_start == 42
c3 = c.copy()
assert c3.get_state() != c.get_state()
c.id = c3.id = "foo"
assert c3.get_state() == c.get_state()
class TestServer:
def test_basic(self):
s = context.Server(("address", 22))
assert repr(s)
assert str(s)
s.timestamp_tls_setup = 1607780791
assert str(s)
s.alpn = b"foo"
s.sockname = ("127.0.0.1", 54321)
assert str(s) == "Server(address:22, state=closed, alpn=foo, src_port=54321)"
def test_state(self):
c = tflow.tserver_conn()
c2 = c.copy()
assert c2.get_state() != c.get_state()
c.id = c2.id = "foo"
assert c2.get_state() == c.get_state()
def test_context():
with taddons.context() as tctx:
c = context.Context(

7
test/mitmproxy/proxy/test_events.py
View File

@ -2,12 +2,13 @@ from unittest.mock import Mock
import pytest
from mitmproxy.proxy import events, context, commands
from mitmproxy import connection
from mitmproxy.proxy import events, commands
@pytest.fixture
def tconn() -> context.Server:
return context.Server(None)
def tconn() -> connection.Server:
return connection.Server(None)
def test_dataclasses(tconn):

3
test/mitmproxy/proxy/test_tunnel.py
View File

@ -4,7 +4,8 @@ import pytest
from mitmproxy.proxy import tunnel, layer
from mitmproxy.proxy.commands import SendData, Log, CloseConnection, OpenConnection
from mitmproxy.proxy.context import Context, Server, ConnectionState
from mitmproxy.connection import Server, ConnectionState
from mitmproxy.proxy.context import Context
from mitmproxy.proxy.events import Event, DataReceived, Start, ConnectionClosed
from test.mitmproxy.proxy.tutils import Playbook, reply

2
test/mitmproxy/proxy/tutils.py
View File

@ -7,7 +7,7 @@ import typing
from mitmproxy.proxy import commands, context, layer
from mitmproxy.proxy import events
from mitmproxy.proxy.context import ConnectionState
from mitmproxy.connection import ConnectionState
from mitmproxy.proxy.events import command_reply_subclasses
from mitmproxy.proxy.layer import Layer

78
test/mitmproxy/test_connection.py Normal file
View File

@ -0,0 +1,78 @@
from mitmproxy.connection import Server, Client, ConnectionState
from mitmproxy.test.tflow import tclient_conn, tserver_conn
class TestConnection:
def test_basic(self):
c = Client(
("127.0.0.1", 52314),
("127.0.0.1", 8080),
1607780791
)
assert not c.tls_established
c.timestamp_tls_setup = 1607780792
assert c.tls_established
assert c.connected
c.state = ConnectionState.CAN_WRITE
assert not c.connected
def test_eq(self):
c = tclient_conn()
c2 = c.copy()
assert c == c
assert c != c2
assert c != 42
assert hash(c) != hash(c2)
c2.id = c.id
assert c == c2
class TestClient:
def test_basic(self):
c = Client(
("127.0.0.1", 52314),
("127.0.0.1", 8080),
1607780791
)
assert repr(c)
assert str(c)
c.timestamp_tls_setup = 1607780791
assert str(c)
c.alpn = b"foo"
assert str(c) == "Client(127.0.0.1:52314, state=open, alpn=foo)"
def test_state(self):
c = tclient_conn()
assert Client.from_state(c.get_state()).get_state() == c.get_state()
c2 = tclient_conn()
assert c != c2
c2.timestamp_start = 42
c.set_state(c2.get_state())
assert c.timestamp_start == 42
c3 = c.copy()
assert c3.get_state() != c.get_state()
c.id = c3.id = "foo"
assert c3.get_state() == c.get_state()
class TestServer:
def test_basic(self):
s = Server(("address", 22))
assert repr(s)
assert str(s)
s.timestamp_tls_setup = 1607780791
assert str(s)
s.alpn = b"foo"
s.sockname = ("127.0.0.1", 54321)
assert str(s) == "Server(address:22, state=closed, alpn=foo, src_port=54321)"
def test_state(self):
c = tserver_conn()
c2 = c.copy()
assert c2.get_state() != c.get_state()
c.id = c2.id = "foo"
assert c2.get_state() == c.get_state()

1011
test/mitmproxy/test_http.py
View File

File diff suppressed because it is too large Load Diff