diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 6c0f481b6..3a708c7c9 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -1,7 +1,9 @@ from __future__ import absolute_import + import urwid import urwid.util import os + from .. import utils from ..protocol.http import CONTENT_MISSING, decoded @@ -284,15 +286,16 @@ def ask_copy_part(scope, flow, master, state): def ask_save_body(part, master, state, flow): """ - Save either the request or the response body to disk. - part can either be "q" (request), "s" (response) or None (ask user if necessary). + Save either the request or the response body to disk. part can either be + "q" (request), "s" (response) or None (ask user if necessary). """ request_has_content = flow.request and flow.request.content response_has_content = flow.response and flow.response.content if part is None: - # We first need to determine whether we want to save the request or the response content. + # We first need to determine whether we want to save the request or the + # response content. if request_has_content and response_has_content: master.prompt_onekey( "Save", @@ -311,9 +314,19 @@ def ask_save_body(part, master, state, flow): ask_save_body("q", master, state, flow) elif part == "q" and request_has_content: - ask_save_path("Save request content: ", flow.request.get_decoded_content(), master, state) + ask_save_path( + "Save request content: ", + flow.request.get_decoded_content(), + master, + state + ) elif part == "s" and response_has_content: - ask_save_path("Save response content: ", flow.response.get_decoded_content(), master, state) + ask_save_path( + "Save response content: ", + flow.response.get_decoded_content(), + master, + state + ) else: master.statusbar.message("No content to save.") diff --git a/libmproxy/console/contentview.py b/libmproxy/console/contentview.py index 582723bb1..95d908a45 100644 --- a/libmproxy/console/contentview.py +++ b/libmproxy/console/contentview.py @@ -1,14 +1,23 @@ from __future__ import absolute_import -import logging, subprocess, re, cStringIO, traceback, json, urwid +import cStringIO +import json +import logging +import lxml.html +import lxml.etree from PIL import Image from PIL.ExifTags import TAGS +import re +import subprocess +import traceback +import urwid -import lxml.html, lxml.etree import netlib.utils + from . import common from .. import utils, encoding, flow from ..contrib import jsbeautifier, html2text from ..contrib.wbxml.ASCommandResponse import ASCommandResponse + try: import pyamf from pyamf import remoting, flex @@ -62,6 +71,7 @@ class ViewAuto: name = "Auto" prompt = ("auto", "a") content_types = [] + def __call__(self, hdrs, content, limit): ctype = hdrs.get_first("content-type") if ctype: @@ -78,6 +88,7 @@ class ViewRaw: name = "Raw" prompt = ("raw", "r") content_types = [] + def __call__(self, hdrs, content, limit): txt = _view_text(content[:limit], len(content), limit) return "Raw", txt @@ -87,6 +98,7 @@ class ViewHex: name = "Hex" prompt = ("hex", "e") content_types = [] + def __call__(self, hdrs, content, limit): txt = [] for offset, hexa, s in netlib.utils.hexdump(content[:limit]): @@ -105,8 +117,14 @@ class ViewXML: name = "XML" prompt = ("xml", "x") content_types = ["text/xml"] + def __call__(self, hdrs, content, limit): - parser = lxml.etree.XMLParser(remove_blank_text=True, resolve_entities=False, strip_cdata=False, recover=False) + parser = lxml.etree.XMLParser( + remove_blank_text=True, + resolve_entities=False, + strip_cdata=False, + recover=False + ) try: document = lxml.etree.fromstring(content, parser) except lxml.etree.XMLSyntaxError: @@ -121,18 +139,18 @@ class ViewXML: lxml.etree.tostring(p) ) p = p.getprevious() - doctype=docinfo.doctype + doctype = docinfo.doctype if prev: doctype += "\n".join(prev).strip() doctype = doctype.strip() s = lxml.etree.tostring( - document, - pretty_print=True, - xml_declaration=True, - doctype=doctype or None, - encoding = docinfo.encoding - ) + document, + pretty_print=True, + xml_declaration=True, + doctype=doctype or None, + encoding = docinfo.encoding + ) txt = [] for i in s[:limit].strip().split("\n"): @@ -147,6 +165,7 @@ class ViewJSON: name = "JSON" prompt = ("json", "s") content_types = ["application/json"] + def __call__(self, hdrs, content, limit): lines = utils.pretty_json(content) if lines: @@ -167,12 +186,20 @@ class ViewHTML: name = "HTML" prompt = ("html", "h") content_types = ["text/html"] + def __call__(self, hdrs, content, limit): if utils.isXML(content): - parser = lxml.etree.HTMLParser(strip_cdata=True, remove_blank_text=True) + parser = lxml.etree.HTMLParser( + strip_cdata=True, + remove_blank_text=True + ) d = lxml.html.fromstring(content, parser=parser) docinfo = d.getroottree().docinfo - s = lxml.etree.tostring(d, pretty_print=True, doctype=docinfo.doctype) + s = lxml.etree.tostring( + d, + pretty_print=True, + doctype=docinfo.doctype + ) return "HTML", _view_text(s[:limit], len(s), limit) @@ -180,6 +207,7 @@ class ViewHTMLOutline: name = "HTML Outline" prompt = ("html outline", "o") content_types = ["text/html"] + def __call__(self, hdrs, content, limit): content = content.decode("utf-8") h = html2text.HTML2Text(baseurl="") @@ -194,14 +222,15 @@ class ViewURLEncoded: name = "URL-encoded" prompt = ("urlencoded", "u") content_types = ["application/x-www-form-urlencoded"] + def __call__(self, hdrs, content, limit): lines = utils.urldecode(content) if lines: body = common.format_keyvals( - [(k+":", v) for (k, v) in lines], - key = "header", - val = "text" - ) + [(k+":", v) for (k, v) in lines], + key = "header", + val = "text" + ) return "URLEncoded form", body @@ -209,6 +238,7 @@ class ViewMultipart: name = "Multipart Form" prompt = ("multipart", "m") content_types = ["multipart/form-data"] + def __call__(self, hdrs, content, limit): v = hdrs.get_first("content-type") if v: @@ -322,12 +352,14 @@ class ViewJavaScript: "application/javascript", "text/javascript" ] + def __call__(self, hdrs, content, limit): opts = jsbeautifier.default_options() opts.indent_size = 2 res = jsbeautifier.beautify(content[:limit], opts) return "JavaScript", _view_text(res, len(res), limit) + class ViewCSS: name = "CSS" prompt = ("css", "c") @@ -355,6 +387,7 @@ class ViewImage: "image/vnd.microsoft.icon", "image/x-icon", ] + def __call__(self, hdrs, content, limit): try: img = Image.open(cStringIO.StringIO(content)) @@ -380,14 +413,17 @@ class ViewImage: ) clean = [] for i in parts: - clean.append([netlib.utils.cleanBin(i[0]), netlib.utils.cleanBin(i[1])]) - fmt = common.format_keyvals( - clean, - key = "header", - val = "text" + clean.append( + [netlib.utils.cleanBin(i[0]), netlib.utils.cleanBin(i[1])] ) + fmt = common.format_keyvals( + clean, + key = "header", + val = "text" + ) return "%s image"%img.format, fmt + class ViewProtobuf: """Human friendly view of protocol buffers The view uses the protoc compiler to decode the binary @@ -403,7 +439,10 @@ class ViewProtobuf: @staticmethod def is_available(): try: - p = subprocess.Popen(["protoc", "--version"], stdout=subprocess.PIPE) + p = subprocess.Popen( + ["protoc", "--version"], + stdout=subprocess.PIPE + ) out, _ = p.communicate() return out.startswith("libprotoc") except: @@ -427,6 +466,7 @@ class ViewProtobuf: txt = _view_text(decoded[:limit], len(decoded), limit) return "Protobuf", txt + class ViewWBXML: name = "WBXML" prompt = ("wbxml", "w") @@ -436,14 +476,14 @@ class ViewWBXML: ] def __call__(self, hdrs, content, limit): - + try: parser = ASCommandResponse(content) parsedContent = parser.xmlString txt = _view_text(parsedContent, len(parsedContent), limit) return "WBXML", txt except: - return None + return None views = [ ViewAuto(), @@ -512,7 +552,7 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc, is_request): # Third-party viewers can fail in unexpected ways... except Exception: s = traceback.format_exc() - s = "Content viewer failed: \n" + s + s = "Content viewer failed: \n" + s logfunc(s, "error") ret = None if not ret: diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index 37bc5ae4a..fe3df5091 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -1,6 +1,10 @@ from __future__ import absolute_import -import copy, re, os + +import copy +import re +import os import urwid + from . import common from .. import utils, filt, script from netlib import http_uastrings @@ -121,7 +125,9 @@ class GridWalker(urwid.ListWalker): try: val = val.decode("string-escape") except ValueError: - self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000) + self.editor.master.statusbar.message( + "Invalid Python-style string encoding.", 1000 + ) return errors = self.lst[self.focus][1] emsg = self.editor.is_error(self.focus_col, val) @@ -155,7 +161,9 @@ class GridWalker(urwid.ListWalker): def start_edit(self): if self.lst: - self.editing = GridRow(self.focus_col, True, self.editor, self.lst[self.focus]) + self.editing = GridRow( + self.focus_col, True, self.editor, self.lst[self.focus] + ) self.editor.master.statusbar.update(footer_editing) self._modified() @@ -187,7 +195,12 @@ class GridWalker(urwid.ListWalker): if self.editing: return self.editing, self.focus elif self.lst: - return GridRow(self.focus_col, False, self.editor, self.lst[self.focus]), self.focus + return GridRow( + self.focus_col, + False, + self.editor, + self.lst[self.focus] + ), self.focus else: return None, None @@ -213,10 +226,13 @@ class GridListBox(urwid.ListBox): FIRST_WIDTH_MAX = 40 FIRST_WIDTH_MIN = 20 + + class GridEditor(urwid.WidgetWrap): title = None columns = None headings = None + def __init__(self, master, value, callback, *cb_args, **cb_kwargs): value = copy.deepcopy(value) self.master, self.value, self.callback = master, value, callback @@ -325,7 +341,9 @@ class GridEditor(urwid.WidgetWrap): self.master.path_prompt("Read file: ", "", self.read_file) elif key == "R": if self.walker.get_current_value() is not None: - self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) + self.master.path_prompt( + "Read unescaped file: ", "", self.read_file, True + ) elif key == "e": o = self.walker.get_current_value() if o is not None: @@ -362,7 +380,9 @@ class GridEditor(urwid.WidgetWrap): ("tab", "next field"), ("enter", "edit field"), ] - text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) + text.extend( + common.format_keyvals(keys, key="key", val="text", indent=4) + ) text.append( urwid.Text( [ @@ -384,6 +404,7 @@ class HeaderEditor(GridEditor): title = "Editing headers" columns = 2 headings = ("Key", "Value") + def make_help(self): h = GridEditor.make_help(self) text = [] @@ -391,7 +412,9 @@ class HeaderEditor(GridEditor): keys = [ ("U", "add User-Agent header"), ] - text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) + text.extend( + common.format_keyvals(keys, key="key", val="text", indent=4) + ) text.append(urwid.Text([("text", "\n")])) text.extend(h) return text @@ -426,6 +449,7 @@ class ReplaceEditor(GridEditor): title = "Editing replacement patterns" columns = 3 headings = ("Filter", "Regex", "Replacement") + def is_error(self, col, val): if col == 0: if not filt.parse(val): @@ -442,6 +466,7 @@ class SetHeadersEditor(GridEditor): title = "Editing header set patterns" columns = 3 headings = ("Filter", "Header", "Value") + def is_error(self, col, val): if col == 0: if not filt.parse(val): @@ -455,7 +480,9 @@ class SetHeadersEditor(GridEditor): keys = [ ("U", "add User-Agent header"), ] - text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) + text.extend( + common.format_keyvals(keys, key="key", val="text", indent=4) + ) text.append(urwid.Text([("text", "\n")])) text.extend(h) return text @@ -491,6 +518,7 @@ class ScriptEditor(GridEditor): title = "Editing scripts" columns = 1 headings = ("Command",) + def is_error(self, col, val): try: script.Script.parse_command(val) diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index 27288a369..fddab5378 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -1,5 +1,7 @@ from __future__ import absolute_import + import urwid + from . import common from .. import filt, version @@ -8,6 +10,7 @@ footer = [ ('heading_key', "q"), ":back ", ] + class HelpView(urwid.ListBox): def __init__(self, master, help_context, state): self.master, self.state = master, state @@ -122,7 +125,9 @@ class HelpView(urwid.ListBox): ("T", "set tcp proxying pattern"), ("u", "set sticky auth expression"), ] - text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) + text.extend( + common.format_keyvals(keys, key="key", val="text", indent=4) + ) text.append(urwid.Text([("head", "\n\nFilter expressions:\n")])) f = [] @@ -167,7 +172,9 @@ class HelpView(urwid.ListBox): ("~q ~b test", "Requests where body contains \"test\""), ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), ] - text.extend(common.format_keyvals(examples, key="key", val="text", indent=4)) + text.extend( + common.format_keyvals(examples, key="key", val="text", indent=4) + ) return text def keypress(self, size, key):