Add the --host option, which uses the value in the Host header for dispaly URLs.

- Can be toggled with "o" then "h" in mitmproxy
- Useful for transparent mode

libmproxy/cmdline.py

@@ -154,6 +154,7 @@ def get_common_options(options):
         script = options.script,
         stickycookie = stickycookie,
         stickyauth = stickyauth,
+        showhost = options.showhost,
         wfile = options.wfile,
         verbosity = options.verbose,
         nopop = options.nopop,
@@ -248,6 +249,11 @@ def common_options(parser):
         help="Byte size limit of HTTP request and response bodies."\
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
+    parser.add_argument(
+        "--host",
+        action="store_true", dest="showhost", default=False,
+        help="Use the Host header to construct URLs for display."
+    )
 
     parser.add_argument(
         "--no-upstream-cert", default=False,

libmproxy/console/__init__.py

@@ -174,6 +174,8 @@ class StatusBar(common.WWrap):
             opts.append("anticache")
         if self.master.anticomp:
             opts.append("anticomp")
+        if self.master.showhost:
+            opts.append("showhost")
         if not self.master.refresh_server_playback:
             opts.append("norefresh")
         if self.master.killextra:
@@ -338,6 +340,7 @@ class Options(object):
         "refresh_server_playback",
         "rfile",
         "script",
+        "showhost",
         "replacements",
         "rheaders",
         "setheaders",
@@ -398,6 +401,7 @@ class ConsoleMaster(flow.FlowMaster):
         self.killextra = options.kill
         self.rheaders = options.rheaders
         self.nopop = options.nopop
+        self.showhost = options.showhost
 
         self.eventlog = options.eventlog
         self.eventlist = urwid.SimpleListWalker([])
@@ -918,6 +922,7 @@ class ConsoleMaster(flow.FlowMaster):
                                         (
                                             ("anticache", "a"),
                                             ("anticomp", "c"),
+                                            ("showhost", "h"),
                                             ("killextra", "k"),
                                             ("norefresh", "n"),
                                             ("no-upstream-certs", "u"),
@@ -957,6 +962,10 @@ class ConsoleMaster(flow.FlowMaster):
             self.anticache = not self.anticache
         if a == "c":
             self.anticomp = not self.anticomp
+        if a == "h":
+            self.showhost = not self.showhost
+            self.sync_list_view()
+            self.refresh_flow(self.currentflow)
         elif a == "k":
             self.killextra = not self.killextra
         elif a == "n":

libmproxy/console/common.py

@@ -177,7 +177,7 @@ class FlowCache:
 flowcache = FlowCache()
 
 
-def format_flow(f, focus, extended=False, padding=2):
+def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     d = dict(
         intercepting = f.intercepting,
 
@@ -185,7 +185,7 @@ def format_flow(f, focus, extended=False, padding=2):
         req_is_replay = f.request.is_replay(),
         req_method = f.request.method,
         req_acked = f.request.reply.acked,
-        req_url = f.request.get_url(),
+        req_url = f.request.get_url(hostheader=hostheader),
 
         err_msg = f.error.msg if f.error else None,
         resp_code = f.response.code if f.response else None,

libmproxy/console/flowlist.py

@@ -105,7 +105,7 @@ class ConnectionItem(common.WWrap):
         common.WWrap.__init__(self, w)
 
     def get_text(self):
-        return common.format_flow(self.flow, self.f)
+        return common.format_flow(self.flow, self.f, hostheader=self.master.showhost)
 
     def selectable(self):
         return True

libmproxy/console/flowview.py

@@ -88,11 +88,11 @@ footer = [
 class FlowViewHeader(common.WWrap):
     def __init__(self, master, f):
         self.master, self.flow = master, f
-        self.w = common.format_flow(f, False, extended=True, padding=0)
+        self.w = common.format_flow(f, False, extended=True, padding=0, hostheader=self.master.showhost)
 
     def refresh_flow(self, f):
         if f == self.flow:
-            self.w = common.format_flow(f, False, extended=True, padding=0)
+            self.w = common.format_flow(f, False, extended=True, padding=0, hostheader=self.master.showhost)
 
 
 class CallbackCache:

libmproxy/console/help.py

@@ -97,6 +97,10 @@ class HelpView(urwid.ListBox):
                     common.highlight_key("anticomp", "c") +
                     [("text", ": prevent compressed responses")]
                 ),
+                (None,
+                    common.highlight_key("showhost", "h") +
+                    [("text", ": use Host header for URL display")]
+                ),
                 (None,
                     common.highlight_key("killextra", "k") +
                     [("text", ": kill requests not part of server replay")]

libmproxy/flow.py

@@ -459,11 +459,19 @@ class Request(HTTPMsg):
         query = utils.urlencode(odict.lst)
         self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]))
 
-    def get_url(self):
+    def get_url(self, hostheader=False):
         """
             Returns a URL string, constructed from the Request's URL compnents.
+
+            If hostheader is True, we use the value specified in the request
+            Host header to construct the URL.
         """
-        return utils.unparse_url(self.scheme, self.host.encode("idna"), self.port, self.path).encode('ascii')
+        if hostheader:
+            host = self.headers.get_first("host") or self.host
+        else:
+            host = self.host
+        host = host.encode("idna")
+        return utils.unparse_url(self.scheme, host, self.port, self.path).encode('ascii')
 
     def set_url(self, url):
         """

test/test_console_common.py

@@ -0,0 +1,10 @@
+import libmproxy.console.common as common
+from libmproxy import utils, flow, encoding
+import tutils
+
+
+def test_format_flow():
+    f = tutils.tflow_full()
+    assert common.format_flow(f, True)
+    assert common.format_flow(f, True, hostheader=True)
+    assert common.format_flow(f, True, extended=True)

test/test_flow.py

@@ -497,7 +497,7 @@ class TestSerialize:
         fm = flow.FlowMaster(None, s)
         fm.load_flows(r)
         assert len(s._flow_list) == 6
-        
+
     def test_filter(self):
         sio = StringIO()
         fl = filt.parse("~c 200")
@@ -783,6 +783,17 @@ class TestRequest:
         r.content = flow.CONTENT_MISSING
         assert not r._assemble()
 
+    def test_get_url(self):
+        h = flow.ODictCaseless()
+        h["test"] = ["test"]
+        c = flow.ClientConnect(("addr", 2222))
+        r = flow.Request(c, (1, 1), "host", 22, "https", "GET", "/", h, "content")
+        assert r.get_url() == "https://host:22/"
+        assert r.get_url(hostheader=True) == "https://host:22/"
+        r.headers["Host"] = ["foo.com"]
+        assert r.get_url() == "https://host:22/"
+        assert r.get_url(hostheader=True) == "https://foo.com:22/"
+
     def test_path_components(self):
         h = flow.ODictCaseless()
         c = flow.ClientConnect(("addr", 2222))