Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-30 03:14:22 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1183 from mitmproxy/module-imports

Browse Source 
use module-level imports only
This commit is contained in:
Aldo Cortesi 2016-06-01 16:26:55 +12:00
commit 92b24c6653
66 changed files with 624 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
mitmproxy/cmdline.py
View File

@ -1,15 +1,17 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import base64
import os
import re
import base64
import configargparse
from netlib.tcp import Address, sslversion_choices
import netlib.http.url
from mitmproxy import filt
from mitmproxy import version
from mitmproxy.proxy import config
from netlib import human
from . import filt, version
from .proxy import config
from netlib import tcp
from netlib.http import url
APP_HOST = "mitm.it"
APP_PORT = 80
@ -104,17 +106,17 @@ def parse_setheader(s):
return _parse_hook(s)
def parse_server_spec(url):
def parse_server_spec(spec):
try:
p = netlib.http.url.parse(url)
p = url.parse(spec)
if p[0] not in ("http", "https"):
raise ValueError()
except ValueError:
raise configargparse.ArgumentTypeError(
"Invalid server specification: %s" % url
"Invalid server specification: %s" % spec
)
address = Address(p[1:3])
address = tcp.Address(p[1:3])
scheme = p[0].lower()
return config.ServerSpec(scheme, address)
@ -477,14 +479,14 @@ def proxy_ssl_options(parser):
group.add_argument(
"--ssl-version-client", dest="ssl_version_client",
default="secure", action="store",
choices=sslversion_choices.keys(),
choices=tcp.sslversion_choices.keys(),
help="Set supported SSL/TLS versions for client connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
)
group.add_argument(
"--ssl-version-server", dest="ssl_version_server",
default="secure", action="store",
choices=sslversion_choices.keys(),
choices=tcp.sslversion_choices.keys(),
help="Set supported SSL/TLS versions for server connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
)

35
mitmproxy/console/__init__.py
View File

@ -1,8 +1,7 @@
from __future__ import absolute_import, print_function
from __future__ import absolute_import, print_function, division
import mailcap
import mimetypes
import tempfile
import os
import os.path
import shlex
@ -10,16 +9,28 @@ import signal
import stat
import subprocess
import sys
import tempfile
import traceback
import urwid
import weakref
from netlib import tcp
import urwid
from .. import flow, script, contentviews, controller
from . import flowlist, flowview, help, window, signals, options
from . import grideditor, palettes, statusbar, palettepicker
from ..exceptions import FlowReadException, ScriptException
from mitmproxy import contentviews
from mitmproxy import controller
from mitmproxy import exceptions
from mitmproxy import flow
from mitmproxy import script
from mitmproxy.console import flowlist
from mitmproxy.console import flowview
from mitmproxy.console import grideditor
from mitmproxy.console import help
from mitmproxy.console import options
from mitmproxy.console import palettepicker
from mitmproxy.console import palettes
from mitmproxy.console import signals
from mitmproxy.console import statusbar
from mitmproxy.console import window
from netlib import tcp
EVENTLOG_SIZE = 500
@ -231,7 +242,7 @@ class ConsoleMaster(flow.FlowMaster):
for i in options.scripts:
try:
self.load_script(i)
except ScriptException as e:
except exceptions.ScriptException as e:
print("Script load error: {}".format(e), file=sys.stderr)
sys.exit(1)
@ -352,7 +363,7 @@ class ConsoleMaster(flow.FlowMaster):
"""
try:
return flow.read_flows_from_paths(path)
except FlowReadException as e:
except exceptions.FlowReadException as e:
signals.status_message.send(message=e.strerror)
def client_playback_path(self, path):
@ -636,7 +647,7 @@ class ConsoleMaster(flow.FlowMaster):
reterr = None
try:
flow.FlowMaster.load_flows_file(self, path)
except FlowReadException as e:
except exceptions.FlowReadException as e:
reterr = str(e)
signals.flowlist_change.send(self)
return reterr
@ -666,7 +677,7 @@ class ConsoleMaster(flow.FlowMaster):
for command in commands:
try:
self.load_script(command)
except ScriptException as e:
except exceptions.ScriptException as e:
signals.status_message.send(
message='Error loading "{}".'.format(command)
)

19
mitmproxy/console/common.py
View File

@ -1,18 +1,17 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import os
import urwid
import urwid.util
import os
import netlib
from mitmproxy import flow
from mitmproxy import models
from mitmproxy import utils
from mitmproxy.console import signals
from netlib import human
from .. import utils
from .. import flow
from ..models import decoded
from . import signals
try:
import pyperclip
except:
@ -260,7 +259,7 @@ def copy_flow_format_data(part, scope, flow):
if scope in ("q", "a"):
if flow.request.content is None:
return None, "Request content is missing"
with decoded(flow.request):
with models.decoded(flow.request):
if part == "h":
data += netlib.http.http1.assemble_request(flow.request)
elif part == "c":
@ -273,7 +272,7 @@ def copy_flow_format_data(part, scope, flow):
if scope in ("s", "a") and flow.response:
if flow.response.content is None:
return None, "Response content is missing"
with decoded(flow.response):
with models.decoded(flow.response):
if part == "h":
data += netlib.http.http1.assemble_response(flow.response)
elif part == "c":

8
mitmproxy/console/flowdetailview.py
View File

@ -1,7 +1,9 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import urwid
from . import common, searchable
from .. import utils
from mitmproxy import utils
from mitmproxy.console import common, searchable
def maybe_timestamp(base, attr):

7
mitmproxy/console/flowlist.py
View File

@ -1,9 +1,10 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import urwid
import netlib.http.url
from . import common, signals
from mitmproxy.console import common
from mitmproxy.console import signals
def _mkhelp():

38
mitmproxy/console/flowview.py
View File

@ -1,17 +1,25 @@
from __future__ import absolute_import, division
import os
import traceback
import sys
from __future__ import absolute_import, print_function, division
import math
import os
import sys
import traceback
import urwid
from netlib.http import Headers, status_codes
from . import common, grideditor, signals, searchable, tabs
from . import flowdetailview
from .. import utils, controller, contentviews
from ..models import HTTPRequest, HTTPResponse, decoded
from ..exceptions import ContentViewException
from mitmproxy import contentviews
from mitmproxy import controller
from mitmproxy import exceptions
from mitmproxy import models
from mitmproxy import utils
from mitmproxy.console import common
from mitmproxy.console import flowdetailview
from mitmproxy.console import grideditor
from mitmproxy.console import searchable
from mitmproxy.console import signals
from mitmproxy.console import tabs
from netlib.http import Headers
from netlib.http import status_codes
class SearchError(Exception):
@ -193,12 +201,12 @@ class FlowView(tabs.Tabs):
try:
query = None
if isinstance(message, HTTPRequest):
if isinstance(message, models.HTTPRequest):
query = message.query
description, lines = contentviews.get_content_view(
viewmode, message.content, headers=message.headers, query=query
)
except ContentViewException:
except exceptions.ContentViewException:
s = "Content viewer failed: \n" + traceback.format_exc()
signals.add_event(s, "error")
description, lines = contentviews.get_content_view(
@ -207,7 +215,7 @@ class FlowView(tabs.Tabs):
description = description.replace("Raw", "Couldn't parse: falling back to Raw")
# Give hint that you have to tab for the response.
if description == "No content" and isinstance(message, HTTPRequest):
if description == "No content" and isinstance(message, models.HTTPRequest):
description = "No request content (press tab to view response)"
# If the users has a wide terminal, he gets fewer lines; this should not be an issue.
@ -372,7 +380,7 @@ class FlowView(tabs.Tabs):
message = self.flow.request
else:
if not self.flow.response:
self.flow.response = HTTPResponse(
self.flow.response = models.HTTPResponse(
self.flow.request.http_version,
200, "OK", Headers(), ""
)
@ -399,7 +407,7 @@ class FlowView(tabs.Tabs):
)
)
if part == "r":
with decoded(message):
with models.decoded(message):
# Fix an issue caused by some editors when editing a
# request/response body. Many editors make it hard to save a
# file without a terminating newline on the last line. When

17
mitmproxy/console/grideditor.py
View File

@ -1,15 +1,18 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import copy
import re
import os
import re
import urwid
from netlib.http import user_agents, cookies
from . import common, signals
from .. import utils, filt, script
from mitmproxy import filt
from mitmproxy import script
from mitmproxy import utils
from mitmproxy.console import common
from mitmproxy.console import signals
from netlib.http import cookies
from netlib.http import user_agents
FOOTER = [
('heading_key', "enter"), ":edit ",

8
mitmproxy/console/help.py
View File

@ -1,9 +1,11 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import urwid
from . import common, signals
from .. import filt, version
from mitmproxy import filt
from mitmproxy import version
from mitmproxy.console import common
from mitmproxy.console import signals
footer = [
("heading", 'mitmproxy v%s ' % version.VERSION),

11
mitmproxy/console/options.py
View File

@ -1,8 +1,13 @@
from __future__ import absolute_import, print_function, division
import urwid
from .. import contentviews
from . import common, signals, grideditor
from . import select, palettes
from mitmproxy import contentviews
from mitmproxy.console import common
from mitmproxy.console import grideditor
from mitmproxy.console import palettes
from mitmproxy.console import select
from mitmproxy.console import signals
footer = [
('heading_key', "enter/space"), ":toggle ",

7
mitmproxy/console/palettepicker.py
View File

@ -1,6 +1,11 @@
from __future__ import absolute_import, print_function, division
import urwid
from . import select, common, palettes, signals
from mitmproxy.console import common
from mitmproxy.console import palettes
from mitmproxy.console import select
from mitmproxy.console import signals
footer = [
('heading_key', "enter/space"), ":select",

1
mitmproxy/console/palettes.py
View File

@ -3,6 +3,7 @@
#
# http://urwid.org/manual/displayattributes.html
#
from __future__ import absolute_import, print_function, division
class Palette:

2
mitmproxy/console/pathedit.py
View File

@ -1,3 +1,5 @@
from __future__ import absolute_import, print_function, division
import glob
import os.path

4
mitmproxy/console/searchable.py
View File

@ -1,6 +1,8 @@
from __future__ import absolute_import, print_function, division
import urwid
from . import signals
from mitmproxy.console import signals
class Highlight(urwid.AttrMap):

4
mitmproxy/console/select.py
View File

@ -1,6 +1,8 @@
from __future__ import absolute_import, print_function, division
import urwid
from . import common
from mitmproxy.console import common
class _OptionWidget(urwid.WidgetWrap):

2
mitmproxy/console/signals.py
View File

@ -1,3 +1,5 @@
from __future__ import absolute_import, print_function, division
import blinker
# Show a status message in the action bar

6
mitmproxy/console/statusbar.py
View File

@ -1,10 +1,14 @@
from __future__ import absolute_import, print_function, division
import os.path
import urwid
import netlib.utils
from mitmproxy.console import common
from mitmproxy.console import pathedit
from mitmproxy.console import signals
from netlib import human
from . import pathedit, signals, common
class ActionBar(urwid.WidgetWrap):

2
mitmproxy/console/tabs.py
View File

@ -1,3 +1,5 @@
from __future__ import absolute_import, print_function, division
import urwid

5
mitmproxy/console/window.py
View File

@ -1,5 +1,8 @@
from __future__ import absolute_import, print_function, division
import urwid
from . import signals
from mitmproxy.console import signals
class Window(urwid.Frame):

64
mitmproxy/contentviews.py
View File

@ -12,27 +12,31 @@ use. For HTTP, the message headers are passed as the ``headers`` keyword argumen
requests, the query parameters are passed as the ``query`` keyword argument.
"""
from __future__ import (absolute_import, print_function, division)
from six.moves import cStringIO as StringIO
from __future__ import absolute_import, print_function, division
import datetime
import json
import logging
import subprocess
import sys
import lxml.html
import lxml.etree
import datetime
from PIL import Image
from PIL.ExifTags import TAGS
import html2text
import lxml.etree
import lxml.html
import six
from netlib.odict import ODict
from netlib import encoding, http
from PIL import ExifTags
from PIL import Image
from six.moves import cStringIO as StringIO
import mitmproxy.utils
from mitmproxy import exceptions
from mitmproxy.contrib import jsbeautifier
from mitmproxy.contrib.wbxml import ASCommandResponse
from netlib import encoding
from netlib import http
from netlib import odict
from netlib.http import url
from netlib.utils import clean_bin, hexdump
from . import utils
from .exceptions import ContentViewException
from .contrib import jsbeautifier
from .contrib.wbxml.ASCommandResponse import ASCommandResponse
import netlib.utils
try:
import pyamf
@ -125,11 +129,11 @@ class ViewAuto(View):
ct = "%s/%s" % (ct[0], ct[1])
if ct in content_types_map:
return content_types_map[ct][0](data, **metadata)
elif utils.isXML(data):
elif mitmproxy.utils.isXML(data):
return get("XML")(data, **metadata)
if metadata.get("query"):
return get("Query")(data, **metadata)
if data and utils.isMostlyBin(data):
if data and mitmproxy.utils.isMostlyBin(data):
return get("Hex")(data)
if not data:
return "No content", []
@ -152,7 +156,7 @@ class ViewHex(View):
@staticmethod
def _format(data):
for offset, hexa, s in hexdump(data):
for offset, hexa, s in netlib.utils.hexdump(data):
yield [
("offset", offset + " "),
("text", hexa + " "),
@ -211,7 +215,7 @@ class ViewJSON(View):
content_types = ["application/json"]
def __call__(self, data, **metadata):
pretty_json = utils.pretty_json(data)
pretty_json = mitmproxy.utils.pretty_json(data)
if pretty_json:
return "JSON", format_text(pretty_json)
@ -222,7 +226,7 @@ class ViewHTML(View):
content_types = ["text/html"]
def __call__(self, data, **metadata):
if utils.isXML(data):
if mitmproxy.utils.isXML(data):
parser = lxml.etree.HTMLParser(
strip_cdata=True,
remove_blank_text=True
@ -259,7 +263,7 @@ class ViewURLEncoded(View):
def __call__(self, data, **metadata):
d = url.decode(data)
return "URLEncoded form", format_dict(ODict(d))
return "URLEncoded form", format_dict(odict.ODict(d))
class ViewMultipart(View):
@ -270,7 +274,7 @@ class ViewMultipart(View):
@staticmethod
def _format(v):
yield [("highlight", "Form data:\n")]
for message in format_dict(ODict(v)):
for message in format_dict(odict.ODict(v)):
yield message
def __call__(self, data, **metadata):
@ -415,11 +419,11 @@ class ViewImage(View):
ex = img._getexif()
if ex:
for i in sorted(ex.keys()):
tag = TAGS.get(i, i)
tag = ExifTags.TAGS.get(i, i)
parts.append(
(str(tag), str(ex[i]))
)
fmt = format_dict(ODict(parts))
fmt = format_dict(odict.ODict(parts))
return "%s image" % img.format, fmt
@ -490,7 +494,7 @@ class ViewWBXML(View):
def __call__(self, data, **metadata):
try:
parser = ASCommandResponse(data)
parser = ASCommandResponse.ASCommandResponse(data)
parsedContent = parser.xmlString
if parsedContent:
return "WBXML", format_text(parsedContent)
@ -519,12 +523,12 @@ def add(view):
# TODO: auto-select a different name (append an integer?)
for i in views:
if i.name == view.name:
raise ContentViewException("Duplicate view: " + view.name)
raise exceptions.ContentViewException("Duplicate view: " + view.name)
# TODO: the UI should auto-prompt for a replacement shortcut
for prompt in view_prompts:
if prompt[1] == view.prompt[1]:
raise ContentViewException("Duplicate view shortcut: " + view.prompt[1])
raise exceptions.ContentViewException("Duplicate view shortcut: " + view.prompt[1])
views.append(view)
@ -577,9 +581,9 @@ def safe_to_print(lines, encoding="utf8"):
clean_line = []
for (style, text) in line:
try:
text = clean_bin(text.decode(encoding, "strict"))
text = netlib.utils.clean_bin(text.decode(encoding, "strict"))
except UnicodeDecodeError:
text = clean_bin(text).decode(encoding, "strict")
text = netlib.utils.clean_bin(text).decode(encoding, "strict")
clean_line.append((style, text))
yield clean_line
@ -611,8 +615,8 @@ def get_content_view(viewmode, data, **metadata):
# Third-party viewers can fail in unexpected ways...
except Exception as e:
six.reraise(
ContentViewException,
ContentViewException(str(e)),
exceptions.ContentViewException,
exceptions.ContentViewException(str(e)),
sys.exc_info()[2]
)
if not ret:

12
mitmproxy/controller.py
View File

@ -1,9 +1,11 @@
from __future__ import absolute_import
from six.moves import queue
import threading
import functools
from __future__ import absolute_import, print_function, division
from . import exceptions
import functools
import threading
from six.moves import queue
from mitmproxy import exceptions
Events = frozenset([
"clientconnect",

44
mitmproxy/dump.py
View File

@ -1,13 +1,19 @@
from __future__ import absolute_import, print_function
import traceback
import sys
import click
import itertools
from __future__ import absolute_import, print_function, division
from netlib import tcp, human
from netlib.utils import bytes_to_escaped_str
from . import flow, filt, contentviews, controller
from .exceptions import ContentViewException, FlowReadException, ScriptException
import itertools
import sys
import traceback
import click
from mitmproxy import contentviews
from mitmproxy import controller
from mitmproxy import exceptions
from mitmproxy import filt
from mitmproxy import flow
from netlib import human
from netlib import tcp
from netlib import utils
class DumpError(Exception):
@ -127,13 +133,13 @@ class DumpMaster(flow.FlowMaster):
for command in scripts:
try:
self.load_script(command, use_reloader=True)
except ScriptException as e:
except exceptions.ScriptException as e:
raise DumpError(str(e))
if options.rfile:
try:
self.load_flows_file(options.rfile)
except FlowReadException as v:
except exceptions.FlowReadException as v:
self.add_event("Flow file corrupted.", "error")
raise DumpError(v)
@ -147,7 +153,7 @@ class DumpMaster(flow.FlowMaster):
"""
try:
return flow.read_flows_from_paths(paths)
except FlowReadException as e:
except exceptions.FlowReadException as e:
raise DumpError(str(e))
def add_event(self, e, level="info"):
@ -175,8 +181,8 @@ class DumpMaster(flow.FlowMaster):
if self.o.flow_detail >= 2:
headers = "\r\n".join(
"{}: {}".format(
click.style(bytes_to_escaped_str(k), fg="blue", bold=True),
click.style(bytes_to_escaped_str(v), fg="blue"))
click.style(utils.bytes_to_escaped_str(k), fg="blue", bold=True),
click.style(utils.bytes_to_escaped_str(v), fg="blue"))
for k, v in message.headers.fields
)
self.echo(headers, indent=4)
@ -192,7 +198,7 @@ class DumpMaster(flow.FlowMaster):
message.content,
headers=message.headers
)
except ContentViewException:
except exceptions.ContentViewException:
s = "Content viewer failed: \n" + traceback.format_exc()
self.add_event(s, "debug")
type, lines = contentviews.get_content_view(
@ -238,7 +244,7 @@ class DumpMaster(flow.FlowMaster):
stickycookie = ""
if flow.client_conn:
client = click.style(bytes_to_escaped_str(flow.client_conn.address.host), bold=True)
client = click.style(utils.bytes_to_escaped_str(flow.client_conn.address.host), bold=True)
else:
client = click.style("[replay]", fg="yellow", bold=True)
@ -247,12 +253,12 @@ class DumpMaster(flow.FlowMaster):
GET="green",
DELETE="red"
).get(method.upper(), "magenta")
method = click.style(bytes_to_escaped_str(method), fg=method_color, bold=True)
method = click.style(utils.bytes_to_escaped_str(method), fg=method_color, bold=True)
if self.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
url = click.style(bytes_to_escaped_str(url), bold=True)
url = click.style(utils.bytes_to_escaped_str(url), bold=True)
httpversion = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
@ -282,7 +288,7 @@ class DumpMaster(flow.FlowMaster):
elif 400 <= code < 600:
code_color = "red"
code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
reason = click.style(bytes_to_escaped_str(flow.response.reason), fg=code_color, bold=True)
reason = click.style(utils.bytes_to_escaped_str(flow.response.reason), fg=code_color, bold=True)
if flow.response.content is None:
size = "(content missing)"

5
mitmproxy/exceptions.py
View File

@ -5,11 +5,10 @@ Every Exception mitmproxy raises shall be a subclass of ProxyException.
See also: http://lucumr.pocoo.org/2014/10/16/on-error-handling/
"""
from __future__ import (absolute_import, print_function, division)
import traceback
from __future__ import absolute_import, print_function, division
import sys
import traceback
class ProxyException(Exception):

4
mitmproxy/filt.py
View File

@ -31,9 +31,11 @@
~c CODE Response code.
rex Equivalent to ~u rex
"""
from __future__ import absolute_import, print_function
from __future__ import absolute_import, print_function, division
import re
import sys
import pyparsing as pp

2
mitmproxy/flow/export.py
View File

@ -1,3 +1,5 @@
from __future__ import absolute_import, print_function, division
import json
import re
from textwrap import dedent

15
mitmproxy/flow/io.py
View File

@ -1,7 +1,10 @@
from __future__ import absolute_import, print_function, division
import os
from mitmproxy import tnetstring, models
from mitmproxy.exceptions import FlowReadException
from mitmproxy import exceptions
from mitmproxy import models
from mitmproxy import tnetstring
from mitmproxy.flow import io_compat
@ -38,17 +41,17 @@ class FlowReader:
try:
data = io_compat.migrate_flow(data)
except ValueError as e:
raise FlowReadException(str(e))
raise exceptions.FlowReadException(str(e))
if can_tell:
off = self.fo.tell()
if data["type"] not in models.FLOW_TYPES:
raise FlowReadException("Unknown flow type: {}".format(data["type"]))
raise exceptions.FlowReadException("Unknown flow type: {}".format(data["type"]))
yield models.FLOW_TYPES[data["type"]].from_state(data)
except ValueError:
# Error is due to EOF
if can_tell and self.fo.tell() == off and self.fo.read() == '':
return
raise FlowReadException("Invalid data format.")
raise exceptions.FlowReadException("Invalid data format.")
class FilteredFlowWriter:
@ -79,5 +82,5 @@ def read_flows_from_paths(paths):
with open(path, "rb") as f:
flows.extend(FlowReader(f).stream())
except IOError as e:
raise FlowReadException(e.strerror)
raise exceptions.FlowReadException(e.strerror)
return flows

26
mitmproxy/flow/master.py
View File

@ -1,16 +1,22 @@
from __future__ import absolute_import, print_function, division
import os
import sys
from typing import List, Optional, Set # noqa
from mitmproxy import controller, script, filt, models
from mitmproxy.exceptions import FlowReadException, Kill
from mitmproxy.flow import io, modules
import netlib.exceptions
from mitmproxy import controller
from mitmproxy import exceptions
from mitmproxy import filt
from mitmproxy import models
from mitmproxy import script
from mitmproxy.flow import io
from mitmproxy.flow import modules
from mitmproxy.onboarding import app
from mitmproxy.protocol.http_replay import RequestReplayThread
from mitmproxy.protocol import http_replay
from mitmproxy.proxy.config import HostMatcher
from netlib import utils
from netlib.exceptions import HttpException
class FlowMaster(controller.Master):
@ -311,7 +317,7 @@ class FlowMaster(controller.Master):
freader = io.FlowReader(f)
return self.load_flows(freader)
except IOError as v:
raise FlowReadException(v.strerror)
raise exceptions.FlowReadException(v.strerror)
def process_new_request(self, f):
if self.stickycookie_state:
@ -351,7 +357,7 @@ class FlowMaster(controller.Master):
f.response = None
f.error = None
self.process_new_request(f)
rt = RequestReplayThread(
rt = http_replay.RequestReplayThread(
self.server.config,
f,
self.event_queue if run_scripthooks else False,
@ -405,7 +411,7 @@ class FlowMaster(controller.Master):
)
if err:
self.add_event("Error in wsgi app. %s" % err, "error")
f.reply(Kill)
f.reply(exceptions.Kill)
return
if f not in self.state.flows: # don't add again on replay
self.state.add_flow(f)
@ -421,8 +427,8 @@ class FlowMaster(controller.Master):
try:
if self.stream_large_bodies:
self.stream_large_bodies.run(f, False)
except HttpException:
f.reply(Kill)
except netlib.exceptions.HttpException:
f.reply(exceptions.Kill)
return
self.run_script_hook("responseheaders", f)
return f

9
mitmproxy/flow/modules.py
View File

@ -1,3 +1,5 @@
from __future__ import absolute_import, print_function, division
import collections
import hashlib
import re
@ -5,9 +7,12 @@ import re
from six.moves import http_cookiejar
from six.moves import urllib
from mitmproxy import version, filt, controller
from mitmproxy import controller
from mitmproxy import filt
from mitmproxy import version
from netlib import wsgi
from netlib.http import http1, cookies
from netlib.http import cookies
from netlib.http import http1
class AppRegistry:

5
mitmproxy/flow/state.py
View File

@ -1,9 +1,12 @@
from __future__ import absolute_import, print_function, division
from abc import abstractmethod, ABCMeta
import six
from typing import List # noqa
from mitmproxy import models, filt # noqa
from mitmproxy import filt
from mitmproxy import models # noqa
@six.add_metaclass(ABCMeta)

33
mitmproxy/main.py
View File

@ -1,13 +1,16 @@
from __future__ import print_function, absolute_import
from __future__ import absolute_import, print_function, division
import os
import signal
import sys
from six.moves import _thread # PY3: We only need _thread.error, which is an alias of RuntimeError in 3.3+
from netlib.version_check import check_pyopenssl_version
from . import cmdline
from .exceptions import ServerException
from .proxy.server import DummyServer, ProxyServer
from .proxy.config import process_proxy_options
from mitmproxy import cmdline
from mitmproxy import exceptions
from mitmproxy.proxy import config
from mitmproxy.proxy import server
from netlib import version_check
def assert_utf8_env():
@ -28,11 +31,11 @@ def assert_utf8_env():
def get_server(dummy_server, options):
if dummy_server:
return DummyServer(options)
return server.DummyServer(options)
else:
try:
return ProxyServer(options)
except ServerException as v:
return server.ProxyServer(options)
except exceptions.ServerException as v:
print(str(v), file=sys.stderr)
sys.exit(1)
@ -44,7 +47,7 @@ def mitmproxy(args=None): # pragma: no cover
sys.exit(1)
from . import console
check_pyopenssl_version()
version_check.check_pyopenssl_version()
assert_utf8_env()
parser = cmdline.mitmproxy()
@ -52,7 +55,7 @@ def mitmproxy(args=None): # pragma: no cover
if options.quiet:
options.verbose = 0
proxy_config = process_proxy_options(parser, options)
proxy_config = cmdline.process_proxy_options(parser, options)
console_options = console.Options(**cmdline.get_common_options(options))
console_options.palette = options.palette
console_options.palette_transparent = options.palette_transparent
@ -74,7 +77,7 @@ def mitmproxy(args=None): # pragma: no cover
def mitmdump(args=None): # pragma: no cover
from . import dump
check_pyopenssl_version()
version_check.check_pyopenssl_version()
parser = cmdline.mitmdump()
options = parser.parse_args(args)
@ -82,7 +85,7 @@ def mitmdump(args=None): # pragma: no cover
options.verbose = 0
options.flow_detail = 0
proxy_config = process_proxy_options(parser, options)
proxy_config = config.process_proxy_options(parser, options)
dump_options = dump.Options(**cmdline.get_common_options(options))
dump_options.flow_detail = options.flow_detail
dump_options.keepserving = options.keepserving
@ -108,7 +111,7 @@ def mitmdump(args=None): # pragma: no cover
def mitmweb(args=None): # pragma: no cover
from . import web
check_pyopenssl_version()
version_check.check_pyopenssl_version()
parser = cmdline.mitmweb()
@ -116,7 +119,7 @@ def mitmweb(args=None): # pragma: no cover
if options.quiet:
options.verbose = 0
proxy_config = process_proxy_options(parser, options)
proxy_config = config.process_proxy_options(parser, options)
web_options = web.Options(**cmdline.get_common_options(options))
web_options.intercept = options.intercept
web_options.wdebug = options.wdebug

8
mitmproxy/models/__init__.py
View File

@ -1,12 +1,12 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from netlib.http import decoded
from .connections import ClientConnection, ServerConnection
from .flow import Flow, Error
from .http import (
HTTPFlow, HTTPRequest, HTTPResponse, Headers,
make_error_response, make_connect_request, make_connect_response, expect_continue_response
)
from netlib.http import decoded
from .connections import ClientConnection, ServerConnection
from .flow import Flow, Error
from .tcp import TCPFlow
FLOW_TYPES = dict(

8
mitmproxy/models/connections.py
View File

@ -1,12 +1,14 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import copy
import os
import six
from netlib import tcp, certutils
from .. import stateobject, utils
from mitmproxy import stateobject
from mitmproxy import utils
from netlib import certutils
from netlib import tcp
class ClientConnection(tcp.BaseHandler, stateobject.StateObject):

14
mitmproxy/models/flow.py
View File

@ -1,10 +1,14 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import copy
import uuid
from .. import stateobject, utils, version
from .connections import ClientConnection, ServerConnection
from ..exceptions import Kill
from mitmproxy import exceptions
from mitmproxy import stateobject
from mitmproxy import utils
from mitmproxy import version
from mitmproxy.models.connections import ClientConnection
from mitmproxy.models.connections import ServerConnection
class Error(stateobject.StateObject):
@ -151,7 +155,7 @@ class Flow(stateobject.StateObject):
"""
self.error = Error("Connection killed")
self.intercepted = False
self.reply(Kill)
self.reply(exceptions.Kill)
master.error(self)
def intercept(self, master):

12
mitmproxy/models/http.py
View File

@ -1,11 +1,15 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import cgi
from mitmproxy import version
from mitmproxy.models.flow import Flow
from netlib import encoding
from netlib.http import status_codes, Headers, Request, Response
from netlib.http import Headers
from netlib.http import Request
from netlib.http import Response
from netlib.http import status_codes
from netlib.tcp import Address
from .. import version
from .flow import Flow
class MessageMixin(object):

5
mitmproxy/models/tcp.py
View File

@ -1,8 +1,11 @@
from __future__ import absolute_import, print_function, division
import time
from typing import List
import netlib.basetypes
from .flow import Flow
from mitmproxy.models.flow import Flow
class TCPMessage(netlib.basetypes.Serializable):

11
mitmproxy/onboarding/app.py
View File

@ -1,12 +1,13 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import os
import tornado.template
import tornado.web
import tornado.wsgi
import tornado.template
from .. import utils
from ..proxy import config
from mitmproxy import utils
from mitmproxy.proxy import config
loader = tornado.template.Loader(utils.pkg_data.path("onboarding/templates"))

1
mitmproxy/platform/osx.py
View File

@ -1,4 +1,5 @@
import subprocess
import pf
"""

61
mitmproxy/platform/windows.py
View File

@ -1,18 +1,17 @@
import configargparse
from six.moves import cPickle as pickle
from ctypes import byref, windll, Structure
from ctypes.wintypes import DWORD
import collections
import ctypes
import ctypes.wintypes
import os
import socket
from six.moves import socketserver
import struct
import threading
import time
from collections import OrderedDict
from pydivert.windivert import WinDivert
from pydivert.enum import Direction, Layer, Flag
import configargparse
from pydivert import enum
from pydivert import windivert
from six.moves import cPickle as pickle
from six.moves import socketserver
PROXY_API_PORT = 8085
@ -91,22 +90,22 @@ ERROR_INSUFFICIENT_BUFFER = 0x7A
# http://msdn.microsoft.com/en-us/library/windows/desktop/bb485761(v=vs.85).aspx
class MIB_TCPROW2(Structure):
class MIB_TCPROW2(ctypes.Structure):
_fields_ = [
('dwState', DWORD),
('dwLocalAddr', DWORD),
('dwLocalPort', DWORD),
('dwRemoteAddr', DWORD),
('dwRemotePort', DWORD),
('dwOwningPid', DWORD),
('dwOffloadState', DWORD)
('dwState', ctypes.wintypes.DWORD),
('dwLocalAddr', ctypes.wintypes.DWORD),
('dwLocalPort', ctypes.wintypes.DWORD),
('dwRemoteAddr', ctypes.wintypes.DWORD),
('dwRemotePort', ctypes.wintypes.DWORD),
('dwOwningPid', ctypes.wintypes.DWORD),
('dwOffloadState', ctypes.wintypes.DWORD)
]
# http://msdn.microsoft.com/en-us/library/windows/desktop/bb485772(v=vs.85).aspx
def MIB_TCPTABLE2(size):
class _MIB_TCPTABLE2(Structure):
_fields_ = [('dwNumEntries', DWORD),
class _MIB_TCPTABLE2(ctypes.Structure):
_fields_ = [('dwNumEntries', ctypes.wintypes.DWORD),
('table', MIB_TCPROW2 * size)]
return _MIB_TCPTABLE2()
@ -192,13 +191,13 @@ class TransparentProxy(object):
self.proxy_addr, self.proxy_port = proxy_addr, proxy_port
self.connection_cache_size = cache_size
self.client_server_map = OrderedDict()
self.client_server_map = collections.OrderedDict()
self.api = APIServer(self, (api_host, api_port), APIRequestHandler)
self.api_thread = threading.Thread(target=self.api.serve_forever)
self.api_thread.daemon = True
self.driver = WinDivert()
self.driver = windivert.WinDivert()
self.driver.register()
self.request_filter = custom_filter or " or ".join(
@ -212,7 +211,7 @@ class TransparentProxy(object):
self.addr_pid_map = dict()
self.trusted_pids = set()
self.tcptable2 = MIB_TCPTABLE2(0)
self.tcptable2_size = DWORD(0)
self.tcptable2_size = ctypes.wintypes.DWORD(0)
self.request_local_handle = None
self.request_local_thread = threading.Thread(target=self.request_local)
self.request_local_thread.daemon = True
@ -244,23 +243,23 @@ class TransparentProxy(object):
# real gateway if they are on the same network.
self.icmp_handle = self.driver.open_handle(
filter="icmp",
layer=Layer.NETWORK,
flags=Flag.DROP)
layer=enum.Layer.NETWORK,
flags=enum.Flag.DROP)
self.response_handle = self.driver.open_handle(
filter=self.response_filter,
layer=Layer.NETWORK)
layer=enum.Layer.NETWORK)
self.response_thread.start()
if self.mode == "forward" or self.mode == "both":
self.request_forward_handle = self.driver.open_handle(
filter=self.request_filter,
layer=Layer.NETWORK_FORWARD)
layer=enum.Layer.NETWORK_FORWARD)
self.request_forward_thread.start()
if self.mode == "local" or self.mode == "both":
self.request_local_handle = self.driver.open_handle(
filter=self.request_filter,
layer=Layer.NETWORK)
layer=enum.Layer.NETWORK)
self.request_local_thread.start()
def shutdown(self):
@ -288,9 +287,9 @@ class TransparentProxy(object):
raise
def fetch_pids(self):
ret = windll.iphlpapi.GetTcpTable2(
byref(
self.tcptable2), byref(
ret = ctypes.windll.iphlpapi.GetTcpTable2(
ctypes.byref(
self.tcptable2), ctypes.byref(
self.tcptable2_size), 0)
if ret == ERROR_INSUFFICIENT_BUFFER:
self.tcptable2 = MIB_TCPTABLE2(self.tcptable2_size.value)
@ -352,7 +351,7 @@ class TransparentProxy(object):
self.client_server_map[client] = server
packet.dst_addr, packet.dst_port = self.proxy_addr, self.proxy_port
metadata.direction = Direction.INBOUND
metadata.direction = enum.Direction.INBOUND
packet = self.driver.update_packet_checksums(packet)
# Use any handle thats on the NETWORK layer - request_local may be

9
mitmproxy/protocol/__init__.py
View File

@ -25,15 +25,16 @@ Another subtle design goal of this architecture is that upstream connections sho
as late as possible; this makes server replay without any outgoing connections possible.
"""
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from .base import Layer, ServerConnectionMixin
from .tls import TlsLayer
from .tls import is_tls_record_magic
from .tls import TlsClientHello
from .http import UpstreamConnectLayer
from .http1 import Http1Layer
from .http2 import Http2Layer
from .rawtcp import RawTCPLayer
from .tls import TlsClientHello
from .tls import TlsLayer
from .tls import is_tls_record_magic
__all__ = [
"Layer", "ServerConnectionMixin",

23
mitmproxy/protocol/base.py
View File

@ -1,11 +1,12 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import sys
import six
from ..models import ServerConnection
from ..exceptions import ProtocolException
from netlib.exceptions import TcpException
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import models
class _LayerCodeCompletion(object):
@ -113,7 +114,7 @@ class ServerConnectionMixin(object):
def __init__(self, server_address=None):
super(ServerConnectionMixin, self).__init__()
self.server_conn = ServerConnection(server_address, (self.config.host, 0))
self.server_conn = models.ServerConnection(server_address, (self.config.host, 0))
self.__check_self_connect()
def __check_self_connect(self):
@ -128,7 +129,7 @@ class ServerConnectionMixin(object):
address.host in ("localhost", "127.0.0.1", "::1")
)
if self_connect:
raise ProtocolException(
raise exceptions.ProtocolException(
"Invalid server address: {}\r\n"
"The proxy shall not connect to itself.".format(repr(address))
)
@ -154,7 +155,7 @@ class ServerConnectionMixin(object):
self.server_conn.finish()
self.server_conn.close()
self.channel.tell("serverdisconnect", self.server_conn)
self.server_conn = ServerConnection(address, (source_address.host, 0))
self.server_conn = models.ServerConnection(address, (source_address.host, 0))
def connect(self):
"""
@ -165,15 +166,15 @@ class ServerConnectionMixin(object):
~mitmproxy.exceptions.ProtocolException: if the connection could not be established.
"""
if not self.server_conn.address:
raise ProtocolException("Cannot connect to server, no server address given.")
raise exceptions.ProtocolException("Cannot connect to server, no server address given.")
self.log("serverconnect", "debug", [repr(self.server_conn.address)])
self.channel.ask("serverconnect", self.server_conn)
try:
self.server_conn.connect()
except TcpException as e:
except netlib.exceptions.TcpException as e:
six.reraise(
ProtocolException,
ProtocolException(
exceptions.ProtocolException,
exceptions.ProtocolException(
"Server connection to {} failed: {}".format(
repr(self.server_conn.address), str(e)
)

77
mitmproxy/protocol/http.py
View File

@ -1,30 +1,21 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import sys
import traceback
import h2.exceptions
import six
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import models
from mitmproxy import utils
from mitmproxy.protocol import base
from netlib import http
from netlib import tcp
from netlib.exceptions import HttpException, HttpReadDisconnect, NetlibException
from netlib.http import Headers
from h2.exceptions import H2Error
from .. import utils
from ..exceptions import HttpProtocolException, Http2ProtocolException, ProtocolException
from ..models import (
HTTPFlow,
HTTPResponse,
make_error_response,
make_connect_response,
Error,
expect_continue_response
)
from .base import Layer
class _HttpTransmissionLayer(Layer):
class _HttpTransmissionLayer(base.Layer):
def read_request(self):
raise NotImplementedError()
@ -51,7 +42,7 @@ class _HttpTransmissionLayer(Layer):
def send_response(self, response):
if response.content is None:
raise HttpException("Cannot assemble flow with missing content")
raise netlib.exceptions.HttpException("Cannot assemble flow with missing content")
self.send_response_headers(response)
self.send_response_body(response, [response.content])
@ -89,7 +80,7 @@ class ConnectServerConnection(object):
__nonzero__ = __bool__
class UpstreamConnectLayer(Layer):
class UpstreamConnectLayer(base.Layer):
def __init__(self, ctx, connect_request):
super(UpstreamConnectLayer, self).__init__(ctx)
@ -107,7 +98,7 @@ class UpstreamConnectLayer(Layer):
self.send_request(self.connect_request)
resp = self.read_response(self.connect_request)
if resp.status_code != 200:
raise ProtocolException("Reconnect: Upstream server refuses CONNECT request")
raise exceptions.ProtocolException("Reconnect: Upstream server refuses CONNECT request")
def connect(self):
if not self.server_conn:
@ -129,7 +120,7 @@ class UpstreamConnectLayer(Layer):
self.server_conn.address = address
class HttpLayer(Layer):
class HttpLayer(base.Layer):
def __init__(self, ctx, mode):
super(HttpLayer, self).__init__(ctx)
@ -166,16 +157,16 @@ class HttpLayer(Layer):
self.handle_regular_mode_connect(request)
return
except HttpReadDisconnect:
except netlib.exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except NetlibException as e:
except netlib.exceptions.NetlibException as e:
self.send_error_response(400, repr(e))
six.reraise(ProtocolException, ProtocolException(
six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
try:
flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
flow = models.HTTPFlow(self.client_conn, self.server_conn, live=self)
flow.request = request
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
@ -210,16 +201,16 @@ class HttpLayer(Layer):
self.handle_upstream_mode_connect(flow.request.copy())
return
except (ProtocolException, NetlibException) as e:
except (exceptions.ProtocolException, netlib.exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not flow.response:
flow.error = Error(str(e))
flow.error = models.Error(str(e))
self.channel.ask("error", flow)
self.log(traceback.format_exc(), "debug")
return
else:
six.reraise(ProtocolException, ProtocolException(
six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
finally:
if flow:
@ -229,16 +220,16 @@ class HttpLayer(Layer):
request = self.read_request()
if request.headers.get("expect", "").lower() == "100-continue":
# TODO: We may have to use send_response_headers for HTTP2 here.
self.send_response(expect_continue_response)
self.send_response(models.expect_continue_response)
request.headers.pop("expect")
request.body = b"".join(self.read_request_body(request))
return request
def send_error_response(self, code, message):
try:
response = make_error_response(code, message)
response = models.make_error_response(code, message)
self.send_response(response)
except (NetlibException, H2Error, Http2ProtocolException):
except (netlib.exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException):
self.log(traceback.format_exc(), "debug")
def change_upstream_proxy_server(self, address):
@ -249,7 +240,7 @@ class HttpLayer(Layer):
def handle_regular_mode_connect(self, request):
self.set_server((request.host, request.port))
self.send_response(make_connect_response(request.data.http_version))
self.send_response(models.make_connect_response(request.data.http_version))
layer = self.ctx.next_layer(self)
layer()
@ -283,7 +274,7 @@ class HttpLayer(Layer):
try:
get_response()
except NetlibException as e:
except netlib.exceptions.NetlibException as e:
self.log(
"server communication error: %s" % repr(e),
level="debug"
@ -300,9 +291,9 @@ class HttpLayer(Layer):
# > read (100-n)% of large request
# > send large request upstream
if isinstance(e, Http2ProtocolException):
if isinstance(e, exceptions.Http2ProtocolException):
# do not try to reconnect for HTTP2
raise ProtocolException("First and only attempt to get response via HTTP2 failed.")
raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.")
self.disconnect()
self.connect()
@ -345,7 +336,7 @@ class HttpLayer(Layer):
flow.request.scheme = "https" if self.__initial_server_tls else "http"
request_reply = self.channel.ask("request", flow)
if isinstance(request_reply, HTTPResponse):
if isinstance(request_reply, models.HTTPResponse):
flow.response = request_reply
return
@ -365,7 +356,7 @@ class HttpLayer(Layer):
if not self.server_conn:
self.connect()
if tls:
raise HttpProtocolException("Cannot change scheme in upstream proxy mode.")
raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.")
"""
# This is a very ugly (untested) workaround to solve a very ugly problem.
if self.server_conn and self.server_conn.tls_established and not ssl:
@ -383,7 +374,7 @@ class HttpLayer(Layer):
def validate_request(self, request):
if request.first_line_format == "absolute" and request.scheme != "http":
raise HttpException("Invalid request scheme: %s" % request.scheme)
raise netlib.exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
expected_request_forms = {
"regular": ("authority", "absolute",),
@ -396,7 +387,7 @@ class HttpLayer(Layer):
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
" or ".join(allowed_request_forms), request.first_line_format
)
raise HttpException(err_message)
raise netlib.exceptions.HttpException(err_message)
if self.mode == "regular" and request.first_line_format == "absolute":
request.first_line_format = "relative"
@ -406,10 +397,10 @@ class HttpLayer(Layer):
if self.config.authenticator.authenticate(request.headers):
self.config.authenticator.clean(request.headers)
else:
self.send_response(make_error_response(
self.send_response(models.make_error_response(
407,
"Proxy Authentication Required",
Headers(**self.config.authenticator.auth_challenge_headers())
http.Headers(**self.config.authenticator.auth_challenge_headers())
))
return False
return True

16
mitmproxy/protocol/http1.py
View File

@ -1,13 +1,11 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from mitmproxy import models
from mitmproxy.protocol import http
from netlib.http import http1
from .http import _HttpTransmissionLayer, HttpLayer
from ..models import HTTPRequest, HTTPResponse
class Http1Layer(_HttpTransmissionLayer):
class Http1Layer(http._HttpTransmissionLayer):
def __init__(self, ctx, mode):
super(Http1Layer, self).__init__(ctx)
@ -15,7 +13,7 @@ class Http1Layer(_HttpTransmissionLayer):
def read_request(self):
req = http1.read_request(self.client_conn.rfile, body_size_limit=self.config.body_size_limit)
return HTTPRequest.wrap(req)
return models.HTTPRequest.wrap(req)
def read_request_body(self, request):
expected_size = http1.expected_http_body_size(request)
@ -27,7 +25,7 @@ class Http1Layer(_HttpTransmissionLayer):
def read_response_headers(self):
resp = http1.read_response_head(self.server_conn.rfile)
return HTTPResponse.wrap(resp)
return models.HTTPResponse.wrap(resp)
def read_response_body(self, request, response):
expected_size = http1.expected_http_body_size(request, response)
@ -63,5 +61,5 @@ class Http1Layer(_HttpTransmissionLayer):
return close_connection
def __call__(self):
layer = HttpLayer(self, self.mode)
layer = http.HttpLayer(self, self.mode)
layer()

86
mitmproxy/protocol/http2.py
View File

@ -1,29 +1,27 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import threading
import time
import traceback
import h2.exceptions
import hyperframe
import six
from h2 import connection
from h2 import events
from six.moves import queue
import traceback
import six
from h2.connection import H2Connection
from h2.exceptions import StreamClosedError
from h2 import events
from hyperframe.frame import PriorityFrame
from netlib.tcp import ssl_read_select
from netlib.exceptions import HttpException
from netlib.http import Headers
from netlib.http.http2 import framereader
import netlib.http.url
from .base import Layer
from .http import _HttpTransmissionLayer, HttpLayer
from ..exceptions import ProtocolException, Http2ProtocolException
from ..models import HTTPRequest, HTTPResponse
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import models
from mitmproxy.protocol import base
from mitmproxy.protocol import http
import netlib.http
from netlib import tcp
from netlib.http import http2
class SafeH2Connection(H2Connection):
class SafeH2Connection(connection.H2Connection):
def __init__(self, conn, *args, **kwargs):
super(SafeH2Connection, self).__init__(*args, **kwargs)
@ -46,7 +44,7 @@ class SafeH2Connection(H2Connection):
with self.lock:
try:
self.reset_stream(stream_id, error_code)
except StreamClosedError: # pragma: no cover
except h2.exceptions.StreamClosedError: # pragma: no cover
# stream is already closed - good
pass
self.conn.send(self.data_to_send())
@ -59,7 +57,7 @@ class SafeH2Connection(H2Connection):
def safe_send_headers(self, is_zombie, stream_id, headers):
with self.lock:
if is_zombie(): # pragma: no cover
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
self.send_headers(stream_id, headers.fields)
self.conn.send(self.data_to_send())
@ -70,7 +68,7 @@ class SafeH2Connection(H2Connection):
self.lock.acquire()
if is_zombie(): # pragma: no cover
self.lock.release()
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
max_outbound_frame_size = self.max_outbound_frame_size
frame_chunk = chunk[position:position + max_outbound_frame_size]
if self.local_flow_control_window(stream_id) < len(frame_chunk):
@ -83,12 +81,12 @@ class SafeH2Connection(H2Connection):
position += max_outbound_frame_size
with self.lock:
if is_zombie(): # pragma: no cover
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
self.end_stream(stream_id)
self.conn.send(self.data_to_send())
class Http2Layer(Layer):
class Http2Layer(base.Layer):
def __init__(self, ctx, mode):
super(Http2Layer, self).__init__(ctx)
@ -108,13 +106,13 @@ class Http2Layer(Layer):
self.active_conns.append(self.server_conn.connection)
def connect(self): # pragma: no cover
raise Http2ProtocolException("HTTP2 layer should already have a connection.")
raise exceptions.Http2ProtocolException("HTTP2 layer should already have a connection.")
def set_server(self): # pragma: no cover
raise Http2ProtocolException("Cannot change server for HTTP2 connections.")
raise exceptions.Http2ProtocolException("Cannot change server for HTTP2 connections.")
def disconnect(self): # pragma: no cover
raise Http2ProtocolException("Cannot dis- or reconnect in HTTP2 connections.")
raise exceptions.Http2ProtocolException("Cannot dis- or reconnect in HTTP2 connections.")
def next_layer(self): # pragma: no cover
# WebSockets over HTTP/2?
@ -135,19 +133,19 @@ class Http2Layer(Layer):
eid = event.stream_id
if isinstance(event, events.RequestReceived):
headers = Headers([[k, v] for k, v in event.headers])
headers = netlib.http.Headers([[k, v] for k, v in event.headers])
self.streams[eid] = Http2SingleStreamLayer(self, eid, headers)
self.streams[eid].timestamp_start = time.time()
self.streams[eid].start()
elif isinstance(event, events.ResponseReceived):
headers = Headers([[k, v] for k, v in event.headers])
headers = netlib.http.Headers([[k, v] for k, v in event.headers])
self.streams[eid].queued_data_length = 0
self.streams[eid].timestamp_start = time.time()
self.streams[eid].response_headers = headers
self.streams[eid].response_arrived.set()
elif isinstance(event, events.DataReceived):
if self.config.body_size_limit and self.streams[eid].queued_data_length > self.config.body_size_limit:
raise HttpException("HTTP body too large. Limit is {}.".format(self.config.body_size_limit))
raise netlib.exceptions.HttpException("HTTP body too large. Limit is {}.".format(self.config.body_size_limit))
self.streams[eid].data_queue.put(event.data)
self.streams[eid].queued_data_length += len(event.data)
source_conn.h2.safe_increment_flow_control(event.stream_id, event.flow_controlled_length)
@ -178,7 +176,7 @@ class Http2Layer(Layer):
self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers)
self.client_conn.send(self.client_conn.h2.data_to_send())
headers = Headers([[str(k), str(v)] for k, v in event.headers])
headers = netlib.http.Headers([[str(k), str(v)] for k, v in event.headers])
headers['x-mitmproxy-pushed'] = 'true'
self.streams[event.pushed_stream_id] = Http2SingleStreamLayer(self, event.pushed_stream_id, headers)
self.streams[event.pushed_stream_id].timestamp_start = time.time()
@ -197,7 +195,7 @@ class Http2Layer(Layer):
depends_on = self.streams[depends_on].server_stream_id
# weight is between 1 and 256 (inclusive), but represented as uint8 (0 to 255)
frame = PriorityFrame(stream_id, depends_on, event.weight - 1, event.exclusive)
frame = hyperframe.frame.PriorityFrame(stream_id, depends_on, event.weight - 1, event.exclusive)
self.server_conn.send(frame.serialize())
elif isinstance(event, events.TrailersReceived):
raise NotImplementedError()
@ -226,7 +224,7 @@ class Http2Layer(Layer):
self.client_conn.send(self.client_conn.h2.data_to_send())
while True:
r = ssl_read_select(self.active_conns, 1)
r = tcp.ssl_read_select(self.active_conns, 1)
for conn in r:
source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn
other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn
@ -234,7 +232,7 @@ class Http2Layer(Layer):
with source_conn.h2.lock:
try:
raw_frame = b''.join(framereader.http2_read_raw_frame(source_conn.rfile))
raw_frame = b''.join(http2.framereader.http2_read_raw_frame(source_conn.rfile))
except:
# read frame failed: connection closed
self._kill_all_streams()
@ -252,7 +250,7 @@ class Http2Layer(Layer):
self._cleanup_streams()
class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
class Http2SingleStreamLayer(http._HttpTransmissionLayer, threading.Thread):
def __init__(self, ctx, stream_id, request_headers):
super(Http2SingleStreamLayer, self).__init__(ctx, name="Thread-Http2SingleStreamLayer-{}".format(stream_id))
@ -336,7 +334,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
data.append(self.request_data_queue.get())
data = b"".join(data)
return HTTPRequest(
return models.HTTPRequest(
first_line_format,
method,
scheme,
@ -361,7 +359,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
with self.server_conn.h2.lock:
# We must not assign a stream id if we are already a zombie.
if self.zombie: # pragma: no cover
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
self.server_stream_id = self.server_conn.h2.get_next_available_stream_id()
self.server_to_client_stream_ids[self.server_stream_id] = self.client_stream_id
@ -382,7 +380,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
message.body
)
if self.zombie: # pragma: no cover
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
def read_response_headers(self):
self.response_arrived.wait()
@ -391,7 +389,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
headers = self.response_headers.copy()
headers.clear(":status")
return HTTPResponse(
return models.HTTPResponse(
http_version=b"HTTP/2.0",
status_code=status_code,
reason='',
@ -412,7 +410,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
yield self.response_data_queue.get()
return
if self.zombie: # pragma: no cover
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
def send_response_headers(self, response):
headers = response.headers.copy()
@ -423,7 +421,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
headers
)
if self.zombie: # pragma: no cover
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
def send_response_body(self, _response, chunks):
self.client_conn.h2.safe_send_body(
@ -432,7 +430,7 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
chunks
)
if self.zombie: # pragma: no cover
raise Http2ProtocolException("Zombie Stream")
raise exceptions.Http2ProtocolException("Zombie Stream")
def check_close_connection(self, flow):
# This layer only handles a single stream.
@ -447,11 +445,11 @@ class Http2SingleStreamLayer(_HttpTransmissionLayer, threading.Thread):
self()
def __call__(self):
layer = HttpLayer(self, self.mode)
layer = http.HttpLayer(self, self.mode)
try:
layer()
except ProtocolException as e:
except exceptions.ProtocolException as e:
self.log(repr(e), "info")
self.log(traceback.format_exc(), "debug")

37
mitmproxy/protocol/http_replay.py
View File

@ -1,13 +1,14 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import threading
import traceback
from mitmproxy.exceptions import ReplayException
from netlib.exceptions import HttpException, TcpException
import netlib.exceptions
from mitmproxy import controller
from mitmproxy import exceptions
from mitmproxy import models
from netlib.http import http1
from ..controller import Channel
from ..models import Error, HTTPResponse, ServerConnection, make_connect_request
from ..exceptions import Kill
# TODO: Doesn't really belong into mitmproxy.protocol...
@ -22,7 +23,7 @@ class RequestReplayThread(threading.Thread):
"""
self.config, self.flow = config, flow
if event_queue:
self.channel = Channel(event_queue, should_exit)
self.channel = controller.Channel(event_queue, should_exit)
else:
self.channel = None
super(RequestReplayThread, self).__init__()
@ -36,17 +37,17 @@ class RequestReplayThread(threading.Thread):
# If we have a channel, run script hooks.
if self.channel:
request_reply = self.channel.ask("request", self.flow)
if isinstance(request_reply, HTTPResponse):
if isinstance(request_reply, models.HTTPResponse):
self.flow.response = request_reply
if not self.flow.response:
# In all modes, we directly connect to the server displayed
if self.config.mode == "upstream":
server_address = self.config.upstream_server.address
server = ServerConnection(server_address, (self.config.host, 0))
server = models.ServerConnection(server_address, (self.config.host, 0))
server.connect()
if r.scheme == "https":
connect_request = make_connect_request((r.host, r.port))
connect_request = models.make_connect_request((r.host, r.port))
server.wfile.write(http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(
@ -55,7 +56,7 @@ class RequestReplayThread(threading.Thread):
body_size_limit=self.config.body_size_limit
)
if resp.status_code != 200:
raise ReplayException("Upstream server refuses CONNECT request")
raise exceptions.ReplayException("Upstream server refuses CONNECT request")
server.establish_ssl(
self.config.clientcerts,
sni=self.flow.server_conn.sni
@ -65,7 +66,7 @@ class RequestReplayThread(threading.Thread):
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = ServerConnection(server_address, (self.config.host, 0))
server = models.ServerConnection(server_address, (self.config.host, 0))
server.connect()
if r.scheme == "https":
server.establish_ssl(
@ -77,20 +78,20 @@ class RequestReplayThread(threading.Thread):
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
self.flow.server_conn = server
self.flow.response = HTTPResponse.wrap(http1.read_response(
self.flow.response = models.HTTPResponse.wrap(http1.read_response(
server.rfile,
r,
body_size_limit=self.config.body_size_limit
))
if self.channel:
response_reply = self.channel.ask("response", self.flow)
if response_reply == Kill:
raise Kill()
except (ReplayException, HttpException, TcpException) as e:
self.flow.error = Error(str(e))
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, netlib.exceptions.NetlibException) as e:
self.flow.error = models.Error(str(e))
if self.channel:
self.channel.ask("error", self.flow)
except Kill:
except exceptions.Kill:
# Kill should only be raised if there's a channel in the
# first place.
from ..proxy.root_context import Log

26
mitmproxy/protocol/rawtcp.py
View File

@ -1,17 +1,17 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import socket
from OpenSSL import SSL
from netlib.exceptions import TcpException
from netlib.tcp import ssl_read_select
from ..models import Error
from ..models.tcp import TCPFlow, TCPMessage
from .base import Layer
import netlib.exceptions
import netlib.tcp
from mitmproxy import models
from mitmproxy.models import tcp
from mitmproxy.protocol import base
class RawTCPLayer(Layer):
class RawTCPLayer(base.Layer):
chunk_size = 4096
def __init__(self, ctx, ignore=False):
@ -22,7 +22,7 @@ class RawTCPLayer(Layer):
self.connect()
if not self.ignore:
flow = TCPFlow(self.client_conn, self.server_conn, self)
flow = models.TCPFlow(self.client_conn, self.server_conn, self)
self.channel.ask("tcp_open", flow)
buf = memoryview(bytearray(self.chunk_size))
@ -33,7 +33,7 @@ class RawTCPLayer(Layer):
try:
while not self.channel.should_exit.is_set():
r = ssl_read_select(conns, 10)
r = netlib.tcp.ssl_read_select(conns, 10)
for conn in r:
dst = server if conn == client else client
@ -52,15 +52,15 @@ class RawTCPLayer(Layer):
return
continue
tcp_message = TCPMessage(dst == server, buf[:size].tobytes())
tcp_message = tcp.TCPMessage(dst == server, buf[:size].tobytes())
if not self.ignore:
flow.messages.append(tcp_message)
self.channel.ask("tcp_message", flow)
dst.sendall(tcp_message.content)
except (socket.error, TcpException, SSL.Error) as e:
except (socket.error, netlib.exceptions.TcpException, SSL.Error) as e:
if not self.ignore:
flow.error = Error("TCP connection closed unexpectedly: {}".format(repr(e)))
flow.error = models.Error("TCP connection closed unexpectedly: {}".format(repr(e)))
self.channel.tell("tcp_error", flow)
finally:
if not self.ignore:

53
mitmproxy/protocol/tls.py
View File

@ -1,16 +1,15 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import struct
import sys
from construct import ConstructError
import construct
import six
from netlib.exceptions import InvalidCertificateException
from netlib.exceptions import TlsException
from ..contrib.tls._constructs import ClientHello
from ..exceptions import ProtocolException, TlsProtocolException, ClientHandshakeException
from .base import Layer
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy.contrib.tls import _constructs
from mitmproxy.protocol import base
# taken from https://testssl.sh/openssl-rfc.mappping.html
@ -246,11 +245,11 @@ def get_client_hello(client_conn):
while len(client_hello) < client_hello_size:
record_header = client_conn.rfile.peek(offset + 5)[offset:]
if not is_tls_record_magic(record_header) or len(record_header) != 5:
raise TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header)
raise exceptions.TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header)
record_size = struct.unpack("!H", record_header[3:])[0] + 5
record_body = client_conn.rfile.peek(offset + record_size)[offset + 5:]
if len(record_body) != record_size - 5:
raise TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body)
raise exceptions.TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body)
client_hello += record_body
offset += record_size
client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4
@ -260,7 +259,7 @@ def get_client_hello(client_conn):
class TlsClientHello(object):
def __init__(self, raw_client_hello):
self._client_hello = ClientHello.parse(raw_client_hello)
self._client_hello = _constructs.ClientHello.parse(raw_client_hello)
def raw(self):
return self._client_hello
@ -297,21 +296,23 @@ class TlsClientHello(object):
"""
try:
raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header.
except ProtocolException as e:
raise TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e))
except exceptions.ProtocolException as e:
raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e))
try:
return cls(raw_client_hello)
except ConstructError as e:
raise TlsProtocolException('Cannot parse Client Hello: %s, Raw Client Hello: %s' %
(repr(e), raw_client_hello.encode("hex")))
except construct.ConstructError as e:
raise exceptions.TlsProtocolException(
'Cannot parse Client Hello: %s, Raw Client Hello: %s' %
(repr(e), raw_client_hello.encode("hex"))
)
def __repr__(self):
return "TlsClientHello( sni: %s alpn_protocols: %s, cipher_suites: %s)" % \
(self.sni, self.alpn_protocols, self.cipher_suites)
class TlsLayer(Layer):
class TlsLayer(base.Layer):
"""
The TLS layer implements transparent TLS connections.
@ -345,7 +346,7 @@ class TlsLayer(Layer):
# Peek into the connection, read the initial client hello and parse it to obtain SNI and ALPN values.
try:
self._client_hello = TlsClientHello.from_client_conn(self.client_conn)
except TlsProtocolException as e:
except exceptions.TlsProtocolException as e:
self.log("Cannot parse Client Hello: %s" % repr(e), "error")
# Do we need to do a server handshake now?
@ -490,10 +491,10 @@ class TlsLayer(Layer):
# The reason for this might be difficult to find, so we try to peek here to see if it
# raises ann error.
self.client_conn.rfile.peek(1)
except TlsException as e:
except netlib.exceptions.TlsException as e:
six.reraise(
ClientHandshakeException,
ClientHandshakeException(
exceptions.ClientHandshakeException,
exceptions.ClientHandshakeException(
"Cannot establish TLS with client (sni: {sni}): {e}".format(
sni=self._client_hello.sni, e=repr(e)
),
@ -544,7 +545,7 @@ class TlsLayer(Layer):
(tls_cert_err['depth'], tls_cert_err['errno']),
"error")
self.log("Ignoring server verification error, continuing with connection", "error")
except InvalidCertificateException as e:
except netlib.exceptions.InvalidCertificateException as e:
tls_cert_err = self.server_conn.ssl_verification_error
self.log(
"TLS verification failed for upstream server at depth %s with error: %s" %
@ -552,18 +553,18 @@ class TlsLayer(Layer):
"error")
self.log("Aborting connection attempt", "error")
six.reraise(
TlsProtocolException,
TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
exceptions.TlsProtocolException,
exceptions.TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
address=repr(self.server_conn.address),
sni=self.server_sni,
e=repr(e),
)),
sys.exc_info()[2]
)
except TlsException as e:
except netlib.exceptions.TlsException as e:
six.reraise(
TlsProtocolException,
TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
exceptions.TlsProtocolException,
exceptions.TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
address=repr(self.server_conn.address),
sni=self.server_sni,
e=repr(e),

4
mitmproxy/proxy/__init__.py
View File

@ -1,8 +1,8 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from .server import ProxyServer, DummyServer
from .config import ProxyConfig
from .root_context import RootContext, Log
from .server import ProxyServer, DummyServer
__all__ = [
"ProxyServer", "DummyServer",

17
mitmproxy/proxy/config.py
View File

@ -1,4 +1,5 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import collections
import os
import re
@ -6,11 +7,11 @@ import re
import six
from OpenSSL import SSL
from netlib import certutils, tcp, human
from mitmproxy import platform
from netlib import certutils
from netlib import human
from netlib import tcp
from netlib.http import authentication
from netlib.tcp import Address, sslversion_choices
from .. import platform
CONF_BASENAME = "mitmproxy"
CA_DIR = "~/.mitmproxy"
@ -91,7 +92,7 @@ class ProxyConfig:
self.body_size_limit = body_size_limit
self.mode = mode
if upstream_server:
self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1]))
self.upstream_server = ServerSpec(upstream_server[0], tcp.Address.wrap(upstream_server[1]))
self.upstream_auth = upstream_auth
else:
self.upstream_server = None
@ -111,9 +112,9 @@ class ProxyConfig:
self.certstore.add_cert_file(spec, cert)
self.openssl_method_client, self.openssl_options_client = \
sslversion_choices[ssl_version_client]
tcp.sslversion_choices[ssl_version_client]
self.openssl_method_server, self.openssl_options_server = \
sslversion_choices[ssl_version_server]
tcp.sslversion_choices[ssl_version_server]
if ssl_verify_upstream_cert:
self.openssl_verification_mode_server = SSL.VERIFY_PEER

3
mitmproxy/proxy/modes/__init__.py
View File

@ -1,4 +1,5 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from .http_proxy import HttpProxy, HttpUpstreamProxy
from .reverse_proxy import ReverseProxy
from .socks_proxy import Socks5Proxy

8
mitmproxy/proxy/modes/http_proxy.py
View File

@ -1,9 +1,9 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from ...protocol import Layer, ServerConnectionMixin
from mitmproxy import protocol
class HttpProxy(Layer, ServerConnectionMixin):
class HttpProxy(protocol.Layer, protocol.ServerConnectionMixin):
def __call__(self):
layer = self.ctx.next_layer(self)
@ -14,7 +14,7 @@ class HttpProxy(Layer, ServerConnectionMixin):
self.disconnect()
class HttpUpstreamProxy(Layer, ServerConnectionMixin):
class HttpUpstreamProxy(protocol.Layer, protocol.ServerConnectionMixin):
def __init__(self, ctx, server_address):
super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address)

6
mitmproxy/proxy/modes/reverse_proxy.py
View File

@ -1,9 +1,9 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from ...protocol import Layer, ServerConnectionMixin
from mitmproxy import protocol
class ReverseProxy(Layer, ServerConnectionMixin):
class ReverseProxy(protocol.Layer, protocol.ServerConnectionMixin):
def __init__(self, ctx, server_address, server_tls):
super(ReverseProxy, self).__init__(ctx, server_address=server_address)

18
mitmproxy/proxy/modes/socks_proxy.py
View File

@ -1,13 +1,13 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from netlib import socks, tcp
from netlib.exceptions import TcpException
from ...exceptions import Socks5ProtocolException
from ...protocol import Layer, ServerConnectionMixin
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import protocol
from netlib import socks
from netlib import tcp
class Socks5Proxy(Layer, ServerConnectionMixin):
class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin):
def __init__(self, ctx):
super(Socks5Proxy, self).__init__(ctx)
@ -51,8 +51,8 @@ class Socks5Proxy(Layer, ServerConnectionMixin):
connect_reply.to_file(self.client_conn.wfile)
self.client_conn.wfile.flush()
except (socks.SocksError, TcpException) as e:
raise Socks5ProtocolException("SOCKS5 mode failure: %s" % repr(e))
except (socks.SocksError, netlib.exceptions.TcpException) as e:
raise exceptions.Socks5ProtocolException("SOCKS5 mode failure: %s" % repr(e))
# https://github.com/mitmproxy/mitmproxy/issues/839
address_bytes = (connect_request.addr.host.encode("idna"), connect_request.addr.port)

12
mitmproxy/proxy/modes/transparent_proxy.py
View File

@ -1,11 +1,11 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from ... import platform
from ...exceptions import ProtocolException
from ...protocol import Layer, ServerConnectionMixin
from mitmproxy import exceptions
from mitmproxy import platform
from mitmproxy import protocol
class TransparentProxy(Layer, ServerConnectionMixin):
class TransparentProxy(protocol.Layer, protocol.ServerConnectionMixin):
def __init__(self, ctx):
super(TransparentProxy, self).__init__(ctx)
@ -15,7 +15,7 @@ class TransparentProxy(Layer, ServerConnectionMixin):
try:
self.server_conn.address = self.resolver.original_addr(self.client_conn.connection)
except Exception as e:
raise ProtocolException("Transparent mode failure: %s" % repr(e))
raise exceptions.ProtocolException("Transparent mode failure: %s" % repr(e))
layer = self.ctx.next_layer(self)
try:

58
mitmproxy/proxy/root_context.py
View File

@ -1,15 +1,13 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import sys
import six
from mitmproxy.exceptions import ProtocolException, TlsProtocolException
from netlib.exceptions import TcpException
from ..protocol import (
RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin,
UpstreamConnectLayer, TlsClientHello
)
from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import protocol
from mitmproxy.proxy import modes
class RootContext(object):
@ -50,53 +48,53 @@ class RootContext(object):
def _next_layer(self, top_layer):
try:
d = top_layer.client_conn.rfile.peek(3)
except TcpException as e:
six.reraise(ProtocolException, ProtocolException(str(e)), sys.exc_info()[2])
client_tls = is_tls_record_magic(d)
except netlib.exceptions.TcpException as e:
six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(str(e)), sys.exc_info()[2])
client_tls = protocol.is_tls_record_magic(d)
# 1. check for --ignore
if self.config.check_ignore:
ignore = self.config.check_ignore(top_layer.server_conn.address)
if not ignore and client_tls:
try:
client_hello = TlsClientHello.from_client_conn(self.client_conn)
except TlsProtocolException as e:
client_hello = protocol.TlsClientHello.from_client_conn(self.client_conn)
except exceptions.TlsProtocolException as e:
self.log("Cannot parse Client Hello: %s" % repr(e), "error")
else:
ignore = self.config.check_ignore((client_hello.sni, 443))
if ignore:
return RawTCPLayer(top_layer, ignore=True)
return protocol.RawTCPLayer(top_layer, ignore=True)
# 2. Always insert a TLS layer, even if there's neither client nor server tls.
# An inline script may upgrade from http to https,
# in which case we need some form of TLS layer.
if isinstance(top_layer, ReverseProxy):
return TlsLayer(top_layer, client_tls, top_layer.server_tls)
if isinstance(top_layer, ServerConnectionMixin) or isinstance(top_layer, UpstreamConnectLayer):
return TlsLayer(top_layer, client_tls, client_tls)
if isinstance(top_layer, modes.ReverseProxy):
return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls)
if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer):
return protocol.TlsLayer(top_layer, client_tls, client_tls)
# 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
if isinstance(top_layer, TlsLayer):
if isinstance(top_layer.ctx, HttpProxy):
return Http1Layer(top_layer, "regular")
if isinstance(top_layer.ctx, HttpUpstreamProxy):
return Http1Layer(top_layer, "upstream")
if isinstance(top_layer, protocol.TlsLayer):
if isinstance(top_layer.ctx, modes.HttpProxy):
return protocol.Http1Layer(top_layer, "regular")
if isinstance(top_layer.ctx, modes.HttpUpstreamProxy):
return protocol.Http1Layer(top_layer, "upstream")
# 4. Check for other TLS cases (e.g. after CONNECT).
if client_tls:
return TlsLayer(top_layer, True, True)
return protocol.TlsLayer(top_layer, True, True)
# 4. Check for --tcp
if self.config.check_tcp(top_layer.server_conn.address):
return RawTCPLayer(top_layer)
return protocol.RawTCPLayer(top_layer)
# 5. Check for TLS ALPN (HTTP1/HTTP2)
if isinstance(top_layer, TlsLayer):
if isinstance(top_layer, protocol.TlsLayer):
alpn = top_layer.client_conn.get_alpn_proto_negotiated()
if alpn == b'h2':
return Http2Layer(top_layer, 'transparent')
return protocol.Http2Layer(top_layer, 'transparent')
if alpn == b'http/1.1':
return Http1Layer(top_layer, 'transparent')
return protocol.Http1Layer(top_layer, 'transparent')
# 6. Check for raw tcp mode
is_ascii = (
@ -105,10 +103,10 @@ class RootContext(object):
all(65 <= x <= 90 and 97 <= x <= 122 for x in six.iterbytes(d))
)
if self.config.rawtcp and not is_ascii:
return RawTCPLayer(top_layer)
return protocol.RawTCPLayer(top_layer)
# 7. Assume HTTP1 by default
return Http1Layer(top_layer, 'transparent')
return protocol.Http1Layer(top_layer, 'transparent')
def log(self, msg, level, subs=()):
"""

57
mitmproxy/proxy/server.py
View File

@ -1,17 +1,18 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import traceback
import sys
import socket
import sys
import traceback
import six
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import models
from mitmproxy.proxy import modes
from mitmproxy.proxy import root_context
from netlib import tcp
from netlib.exceptions import TcpException
from netlib.http.http1 import assemble_response
from ..exceptions import ProtocolException, ServerException, ClientHandshakeException, Kill
from ..models import ClientConnection, make_error_response
from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy
from .root_context import RootContext, Log
from netlib.http import http1
class DummyServer:
@ -43,8 +44,8 @@ class ProxyServer(tcp.TCPServer):
super(ProxyServer, self).__init__((config.host, config.port))
except socket.error as e:
six.reraise(
ServerException,
ServerException('Error starting proxy server: ' + repr(e)),
exceptions.ServerException,
exceptions.ServerException('Error starting proxy server: ' + repr(e)),
sys.exc_info()[2]
)
self.channel = None
@ -67,7 +68,7 @@ class ConnectionHandler(object):
def __init__(self, client_conn, client_address, config, channel):
self.config = config
"""@type: mitmproxy.proxy.config.ProxyConfig"""
self.client_conn = ClientConnection(
self.client_conn = models.ClientConnection(
client_conn,
client_address,
None)
@ -76,7 +77,7 @@ class ConnectionHandler(object):
"""@type: mitmproxy.controller.Channel"""
def _create_root_layer(self):
root_context = RootContext(
root_ctx = root_context.RootContext(
self.client_conn,
self.config,
self.channel
@ -84,25 +85,25 @@ class ConnectionHandler(object):
mode = self.config.mode
if mode == "upstream":
return HttpUpstreamProxy(
root_context,
return modes.HttpUpstreamProxy(
root_ctx,
self.config.upstream_server.address
)
elif mode == "transparent":
return TransparentProxy(root_context)
return modes.TransparentProxy(root_ctx)
elif mode == "reverse":
server_tls = self.config.upstream_server.scheme == "https"
return ReverseProxy(
root_context,
return modes.ReverseProxy(
root_ctx,
self.config.upstream_server.address,
server_tls
)
elif mode == "socks5":
return Socks5Proxy(root_context)
return modes.Socks5Proxy(root_ctx)
elif mode == "regular":
return HttpProxy(root_context)
return modes.HttpProxy(root_ctx)
elif callable(mode): # pragma: no cover
return mode(root_context)
return mode(root_ctx)
else: # pragma: no cover
raise ValueError("Unknown proxy mode: %s" % mode)
@ -114,11 +115,11 @@ class ConnectionHandler(object):
try:
root_layer = self.channel.ask("clientconnect", root_layer)
root_layer()
except Kill:
except exceptions.Kill:
self.log("Connection killed", "info")
except ProtocolException as e:
except exceptions.ProtocolException as e:
if isinstance(e, ClientHandshakeException):
if isinstance(e, exceptions.ClientHandshakeException):
self.log(
"Client Handshake failed. "
"The client may not trust the proxy's certificate for {}.".format(e.server),
@ -133,9 +134,9 @@ class ConnectionHandler(object):
# we send an HTTP error response, which is both
# understandable by HTTP clients and humans.
try:
error_response = make_error_response(502, repr(e))
self.client_conn.send(assemble_response(error_response))
except TcpException:
error_response = models.make_error_response(502, repr(e))
self.client_conn.send(http1.assemble_response(error_response))
except netlib.exceptions.TcpException:
pass
except Exception:
self.log(traceback.format_exc(), "error")
@ -149,4 +150,4 @@ class ConnectionHandler(object):
def log(self, msg, level):
msg = "{}: {}".format(repr(self.client_conn.address), msg)
self.channel.tell("log", Log(msg, level))
self.channel.tell("log", root_context.Log(msg, level))

4
mitmproxy/script/__init__.py
View File

@ -1,8 +1,8 @@
from . import reloader
from .concurrent import concurrent
from .script import Script
from .script_context import ScriptContext
from .concurrent import concurrent
from ..exceptions import ScriptException
from . import reloader
__all__ = [
"Script",

1
mitmproxy/script/concurrent.py
View File

@ -3,6 +3,7 @@ This module provides a @concurrent decorator primitive to
offload computations from mitmproxy's main master thread.
"""
from __future__ import absolute_import, print_function, division
import threading

4
mitmproxy/script/reloader.py
View File

@ -1,6 +1,10 @@
from __future__ import absolute_import, print_function, division
import os
import sys
from watchdog.events import RegexMatchingEventHandler
if sys.platform == 'darwin': # pragma: no cover
from watchdog.observers.polling import PollingObserver as Observer
else:

22
mitmproxy/script/script.py
View File

@ -4,13 +4,15 @@ Script objects know nothing about mitmproxy or mitmproxy's API - this knowledge
by the mitmproxy-specific ScriptContext.
"""
# Do not import __future__ here, this would apply transitively to the inline scripts.
from __future__ import absolute_import, print_function, division
import os
import shlex
import sys
import six
from ..exceptions import ScriptException
from mitmproxy import exceptions
class Script(object):
@ -41,7 +43,7 @@ class Script(object):
@staticmethod
def parse_command(command):
if not command or not command.strip():
raise ScriptException("Empty script command.")
raise exceptions.ScriptException("Empty script command.")
# Windows: escape all backslashes in the path.
if os.name == "nt": # pragma: no cover
backslashes = shlex.split(command, posix=False)[0].count("\\")
@ -49,13 +51,13 @@ class Script(object):
args = shlex.split(command) # pragma: no cover
args[0] = os.path.expanduser(args[0])
if not os.path.exists(args[0]):
raise ScriptException(
raise exceptions.ScriptException(
("Script file not found: %s.\r\n"
"If your script path contains spaces, "
"make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") %
args[0])
elif os.path.isdir(args[0]):
raise ScriptException("Not a file: %s" % args[0])
raise exceptions.ScriptException("Not a file: %s" % args[0])
return args
def load(self):
@ -69,7 +71,7 @@ class Script(object):
ScriptException on failure
"""
if self.ns is not None:
raise ScriptException("Script is already loaded")
raise exceptions.ScriptException("Script is already loaded")
script_dir = os.path.dirname(os.path.abspath(self.args[0]))
self.ns = {'__file__': os.path.abspath(self.args[0])}
sys.path.append(script_dir)
@ -80,8 +82,8 @@ class Script(object):
exec(code, self.ns, self.ns)
except Exception:
six.reraise(
ScriptException,
ScriptException.from_exception_context(),
exceptions.ScriptException,
exceptions.ScriptException.from_exception_context(),
sys.exc_info()[2]
)
finally:
@ -107,15 +109,15 @@ class Script(object):
ScriptException if there was an exception.
"""
if self.ns is None:
raise ScriptException("Script not loaded.")
raise exceptions.ScriptException("Script not loaded.")
f = self.ns.get(name)
if f:
try:
return f(self.ctx, *args, **kwargs)
except Exception:
six.reraise(
ScriptException,
ScriptException.from_exception_context(),
exceptions.ScriptException,
exceptions.ScriptException.from_exception_context(),
sys.exc_info()[2]
)
else:

3
mitmproxy/script/script_context.py
View File

@ -2,7 +2,8 @@
The mitmproxy script context provides an API to inline scripts.
"""
from __future__ import absolute_import, print_function, division
from .. import contentviews
from mitmproxy import contentviews
class ScriptContext(object):

5
mitmproxy/stateobject.py
View File

@ -1,7 +1,8 @@
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import six
from typing import List, Any
from typing import Any
from typing import List
import netlib.basetypes

3
mitmproxy/tnetstring.py
View File

@ -67,9 +67,10 @@ like so::
u'\u03b1'
"""
import six
from collections import deque
import six
__ver_major__ = 0
__ver_minor__ = 2
__ver_patch__ = 0

5
mitmproxy/utils.py
View File

@ -1,7 +1,8 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
import datetime
import time
import json
import time
import netlib.utils

2
mitmproxy/version.py
View File

@ -1,4 +1,4 @@
from __future__ import (absolute_import, print_function, division)
from __future__ import absolute_import, print_function, division
from netlib.version import VERSION, IVERSION

18
mitmproxy/web/__init__.py
View File

@ -1,14 +1,16 @@
from __future__ import absolute_import, print_function
from __future__ import absolute_import, print_function, division
import collections
import tornado.ioloop
import tornado.httpserver
import sys
from netlib.http import authentication
import tornado.httpserver
import tornado.ioloop
from .. import flow, controller
from ..exceptions import FlowReadException
from . import app
from mitmproxy import controller
from mitmproxy import exceptions
from mitmproxy import flow
from mitmproxy.web import app
from netlib.http import authentication
class Stop(Exception):
@ -156,7 +158,7 @@ class WebMaster(flow.FlowMaster):
if options.rfile:
try:
self.load_flows_file(options.rfile)
except FlowReadException as v:
except exceptions.FlowReadException as v:
self.add_event(
"Could not read flow file: %s" % v,
"error"

12
mitmproxy/web/app.py
View File

@ -1,14 +1,16 @@
from __future__ import absolute_import, print_function, division
import base64
import json
import logging
import os.path
import re
import six
import tornado.web
import tornado.websocket
import logging
import json
import base64
from .. import version, filt
from mitmproxy import filt
from mitmproxy import version
def _strip_content(flow_state):

4
netlib/http/http2/__init__.py
View File

@ -1,6 +1,8 @@
from __future__ import absolute_import, print_function, division
from .connections import HTTP2Protocol
from netlib.http.http2 import framereader
__all__ = [
"HTTP2Protocol"
"HTTP2Protocol",
"framereader",
]