From 8ba34be7ab42382ef6e9cd30f2dae48d95e357f0 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Tue, 2 Aug 2016 17:49:50 +0530
Subject: [PATCH 01/27] Goodbye har_extractor

---
 examples/har_extractor.py | 264 --------------------------------------
 1 file changed, 264 deletions(-)
 delete mode 100644 examples/har_extractor.py

diff --git a/examples/har_extractor.py b/examples/har_extractor.py
deleted file mode 100644
index 76059d8ec..000000000
--- a/examples/har_extractor.py
+++ /dev/null
@@ -1,264 +0,0 @@
-"""
-    This inline script utilizes harparser.HAR from
-    https://github.com/JustusW/harparser to generate a HAR log object.
-"""
-import mitmproxy.ctx
-import six
-import sys
-import pytz
-from harparser import HAR
-
-from datetime import datetime
-
-
-class _HARLog(HAR.log):
-    # The attributes need to be registered here for them to actually be
-    # available later via self. This is due to HAREncodable linking __getattr__
-    # to __getitem__. Anything that is set only in __init__ will just be added
-    # as key/value pair to self.__classes__.
-    __page_list__ = []
-    __page_count__ = 0
-    __page_ref__ = {}
-
-    def __init__(self, page_list=[]):
-        self.__page_list__ = page_list
-        self.__page_count__ = 0
-        self.__page_ref__ = {}
-
-        HAR.log.__init__(self, {"version": "1.2",
-                                "creator": {"name": "MITMPROXY HARExtractor",
-                                            "version": "0.1",
-                                            "comment": ""},
-                                "pages": [],
-                                "entries": []})
-
-    def reset(self):
-        self.__init__(self.__page_list__)
-
-    def add(self, obj):
-        if isinstance(obj, HAR.pages):
-            self['pages'].append(obj)
-        if isinstance(obj, HAR.entries):
-            self['entries'].append(obj)
-
-    def create_page_id(self):
-        self.__page_count__ += 1
-        return "autopage_%s" % str(self.__page_count__)
-
-    def set_page_ref(self, page, ref):
-        self.__page_ref__[page] = ref
-
-    def get_page_ref(self, page):
-        return self.__page_ref__.get(page, None)
-
-    def get_page_list(self):
-        return self.__page_list__
-
-
-class Context(object):
-    pass
-
-context = Context()
-
-
-def start():
-    """
-        On start we create a HARLog instance. You will have to adapt this to
-        suit your actual needs of HAR generation. As it will probably be
-        necessary to cluster logs by IPs or reset them from time to time.
-    """
-    if sys.version_info >= (3, 0):
-        raise RuntimeError(
-            "har_extractor.py does not work on Python 3. "
-            "Please check out https://github.com/mitmproxy/mitmproxy/issues/1320 "
-            "if you want to help making this work again."
-        )
-    context.dump_file = None
-    if len(sys.argv) > 1:
-        context.dump_file = sys.argv[1]
-    else:
-        raise ValueError(
-            'Usage: -s "har_extractor.py filename" '
-            '(- will output to stdout, filenames ending with .zhar '
-            'will result in compressed har)'
-        )
-    context.HARLog = _HARLog()
-    context.seen_server = set()
-
-
-def response(flow):
-    """
-       Called when a server response has been received. At the time of this
-       message both a request and a response are present and completely done.
-    """
-    # Values are converted from float seconds to int milliseconds later.
-    ssl_time = -.001
-    connect_time = -.001
-    if flow.server_conn not in context.seen_server:
-        # Calculate the connect_time for this server_conn. Afterwards add it to
-        # seen list, in order to avoid the connect_time being present in entries
-        # that use an existing connection.
-        connect_time = (flow.server_conn.timestamp_tcp_setup -
-                        flow.server_conn.timestamp_start)
-        context.seen_server.add(flow.server_conn)
-
-        if flow.server_conn.timestamp_ssl_setup is not None:
-            # Get the ssl_time for this server_conn as the difference between
-            # the start of the successful tcp setup and the successful ssl
-            # setup. If no ssl setup has been made it is left as -1 since it
-            # doesn't apply to this connection.
-            ssl_time = (flow.server_conn.timestamp_ssl_setup -
-                        flow.server_conn.timestamp_tcp_setup)
-
-    # Calculate the raw timings from the different timestamps present in the
-    # request and response object. For lack of a way to measure it dns timings
-    # can not be calculated. The same goes for HAR blocked: MITMProxy will open
-    # a server connection as soon as it receives the host and port from the
-    # client connection. So the time spent waiting is actually spent waiting
-    # between request.timestamp_end and response.timestamp_start thus it
-    # correlates to HAR wait instead.
-    timings_raw = {
-        'send': flow.request.timestamp_end - flow.request.timestamp_start,
-        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
-        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
-        'connect': connect_time,
-        'ssl': ssl_time
-    }
-
-    # HAR timings are integers in ms, so we have to re-encode the raw timings to
-    # that format.
-    timings = dict([(k, int(1000 * v)) for k, v in six.iteritems(timings_raw)])
-
-    # The full_time is the sum of all timings.
-    # Timings set to -1 will be ignored as per spec.
-    full_time = sum(v for v in timings.values() if v > -1)
-
-    started_date_time = datetime.utcfromtimestamp(
-        flow.request.timestamp_start).replace(tzinfo=pytz.timezone("UTC")).isoformat()
-
-    request_query_string = [{"name": k, "value": v}
-                            for k, v in flow.request.query or {}]
-
-    response_body_size = len(flow.response.content)
-    response_body_decoded_size = len(flow.response.content)
-    response_body_compression = response_body_decoded_size - response_body_size
-
-    entry = HAR.entries({
-        "startedDateTime": started_date_time,
-        "time": full_time,
-        "request": {
-            "method": flow.request.method,
-            "url": flow.request.url,
-            "httpVersion": flow.request.http_version,
-            "cookies": format_cookies(flow.request.cookies),
-            "headers": format_headers(flow.request.headers),
-            "queryString": request_query_string,
-            "headersSize": len(str(flow.request.headers)),
-            "bodySize": len(flow.request.content),
-        },
-        "response": {
-            "status": flow.response.status_code,
-            "statusText": flow.response.reason,
-            "httpVersion": flow.response.http_version,
-            "cookies": format_cookies(flow.response.cookies),
-            "headers": format_headers(flow.response.headers),
-            "content": {
-                "size": response_body_size,
-                "compression": response_body_compression,
-                "mimeType": flow.response.headers.get('Content-Type', '')
-            },
-            "redirectURL": flow.response.headers.get('Location', ''),
-            "headersSize": len(str(flow.response.headers)),
-            "bodySize": response_body_size,
-        },
-        "cache": {},
-        "timings": timings,
-    })
-
-    # If the current url is in the page list of context.HARLog or
-    # does not have a referrer, we add it as a new pages object.
-    is_new_page = (
-        flow.request.url in context.HARLog.get_page_list() or
-        flow.request.headers.get('Referer') is None
-    )
-    if is_new_page:
-        page_id = context.HARLog.create_page_id()
-        context.HARLog.add(
-            HAR.pages({
-                "startedDateTime": entry['startedDateTime'],
-                "id": page_id,
-                "title": flow.request.url,
-                "pageTimings": {}
-            })
-        )
-        context.HARLog.set_page_ref(flow.request.url, page_id)
-        entry['pageref'] = page_id
-
-    # Lookup the referer in the page_ref of context.HARLog to point this entries
-    # pageref attribute to the right pages object, then set it as a new
-    # reference to build a reference tree.
-    elif context.HARLog.get_page_ref(flow.request.headers.get('Referer')) is not None:
-        entry['pageref'] = context.HARLog.get_page_ref(
-            flow.request.headers['Referer']
-        )
-        context.HARLog.set_page_ref(
-            flow.request.headers['Referer'], entry['pageref']
-        )
-
-    context.HARLog.add(entry)
-
-
-def done():
-    """
-        Called once on script shutdown, after any other events.
-    """
-    import pprint
-    import json
-
-    json_dump = context.HARLog.json()
-    compressed_json_dump = context.HARLog.compress()
-
-    if context.dump_file == '-':
-        mitmproxy.ctx.log(pprint.pformat(json.loads(json_dump)))
-    elif context.dump_file.endswith('.zhar'):
-        with open(context.dump_file, "wb") as f:
-            f.write(compressed_json_dump)
-    else:
-        with open(context.dump_file, "wb") as f:
-            f.write(json_dump)
-    mitmproxy.ctx.log(
-        "HAR log finished with %s bytes (%s bytes compressed)" % (
-            len(json_dump), len(compressed_json_dump)
-        )
-    )
-    mitmproxy.ctx.log(
-        "Compression rate is %s%%" % str(
-            100. * len(compressed_json_dump) / len(json_dump)
-        )
-    )
-
-
-def format_cookies(obj):
-    if obj:
-        return [{"name": k.strip(), "value": v[0]} for k, v in obj.items()]
-    return ""
-
-
-def format_headers(obj):
-    if obj:
-        return [{"name": k, "value": v} for k, v in obj.fields]
-    return ""
-
-
-def print_attributes(obj, filter_string=None, hide_privates=False):
-    """
-        Useful helper method to quickly get all attributes of an object and its
-        values.
-    """
-    for attr in dir(obj):
-        if hide_privates and "__" in attr:
-            continue
-        if filter_string is not None and filter_string not in attr:
-            continue
-        value = getattr(obj, attr)
-        print("%s.%s" % ('obj', attr), value, type(value))

From 250e4a17d07004e0de3ffc4c16ef5e8f885c8b38 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Tue, 2 Aug 2016 17:50:14 +0530
Subject: [PATCH 02/27] Welcome har_dump

---
 examples/har_dump.py | 145 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100644 examples/har_dump.py

diff --git a/examples/har_dump.py b/examples/har_dump.py
new file mode 100644
index 000000000..0174da538
--- /dev/null
+++ b/examples/har_dump.py
@@ -0,0 +1,145 @@
+"""
+This inline script can be used to dump flows as HAR files.
+"""
+
+
+import pprint
+import json
+import sys
+
+from datetime import datetime
+import pytz
+
+import mitmproxy
+from mitmproxy import version
+
+HAR = {}
+
+
+def start():
+    """
+        Called once on script startup before any other events.
+    """
+    if len(sys.argv) != 2:
+        raise ValueError(
+            'Usage: -s "har_dump.py filename" '
+            '(- will output to stdout, filenames ending with .zhar '
+            'will result in compressed har)'
+        )
+
+    HAR.update({
+        "log": {
+            "version": "1.2",
+            "creator": {
+                "name": "mitmproxy har_dump",
+                "version": "0.1",
+                "comment": "mitmproxy version %s" % version.MITMPROXY
+            },
+            "pages": [],
+            "entries": []
+        }
+    })
+
+
+def response(flow):
+    """
+       Called when a server response has been received.
+    """
+    entries = HAR["log"]["entries"]
+
+    # TODO: SSL and Connect Timings
+
+    # Calculate raw timings from timestamps.
+
+    # DNS timings can not be calculated for lack of a way to measure it.
+    # The same goes for HAR blocked.
+
+    # mitmproxy will open a server connection as soon as it receives the host
+    # and port from the client connection. So, the time spent waiting is actually
+    # spent waiting between request.timestamp_end and response.timestamp_start thus it
+    # correlates to HAR wait instead.
+    timings_raw = {
+        'send': flow.request.timestamp_end - flow.request.timestamp_start,
+        'receive': flow.response.timestamp_end - flow.response.timestamp_start,
+        'wait': flow.response.timestamp_start - flow.request.timestamp_end,
+    }
+
+    # HAR timings are integers in ms, so we re-encode the raw timings to that format.
+    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
+
+    # full_time is the sum of all timings.
+    # Timings set to -1 will be ignored as per spec.
+    full_time = sum(v for v in timings.values() if v > -1)
+
+    started_date_time = datetime.utcfromtimestamp(
+        flow.request.timestamp_start).replace(tzinfo=pytz.timezone("UTC")).isoformat()
+
+    # Size calculations
+    response_body_size = len(flow.response.content)
+    response_body_decoded_size = len(flow.response.content)
+    response_body_compression = response_body_decoded_size - response_body_size
+
+    entries.append({
+        "startedDateTime": started_date_time,
+        "time": full_time,
+        "request": {
+            "method": flow.request.method,
+            "url": flow.request.url,
+            "httpVersion": flow.request.http_version,
+            "cookies": name_value(flow.request.cookies),
+            "headers": name_value(flow.request.headers),
+            "queryString": name_value(flow.request.query or {}),
+            "headersSize": len(str(flow.request.headers)),
+            "bodySize": len(flow.request.content),
+        },
+        "response": {
+            "status": flow.response.status_code,
+            "statusText": flow.response.reason,
+            "httpVersion": flow.response.http_version,
+            "cookies": name_value(flow.response.cookies),
+            "headers": name_value(flow.response.headers),
+            "content": {
+                "size": response_body_size,
+                "compression": response_body_compression,
+                "mimeType": flow.response.headers.get('Content-Type', '')
+            },
+            "redirectURL": flow.response.headers.get('Location', ''),
+            "headersSize": len(str(flow.response.headers)),
+            "bodySize": response_body_size,
+        },
+        "cache": {},
+        "timings": timings,
+    })
+
+
+def done():
+    """
+        Called once on script shutdown, after any other events.
+    """
+    dump_file = sys.argv[1]
+
+    if dump_file == '-':
+        mitmproxy.ctx.log(pprint.pformat(HAR))
+    # TODO: .zhar compression
+    else:
+        with open(dump_file, "wb") as f:
+            f.write(json.dumps(HAR, indent=2))
+
+    # TODO: Log results via mitmproxy.ctx.log
+
+
+def name_value(obj):
+    """
+        Convert (key, value) pairs to HAR format.
+    """
+
+    items = []
+    if hasattr(obj, 'fields'):
+        items = obj.fields
+    elif hasattr(obj, 'items'):
+        items = obj.items()
+
+    if items:
+        return [{"name": k, "value": v} for k, v in items]
+    else:
+        return ""

From 2cabc65f6234fc4ce605862781d464709a3c6d9d Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 3 Aug 2016 10:57:19 +0530
Subject: [PATCH 03/27] Remove harparser dependency

---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 3aa6abf55..f71a5f106 100644
--- a/setup.py
+++ b/setup.py
@@ -119,7 +119,6 @@ setup(
         ],
         'examples': [
             "beautifulsoup4>=4.4.1, <4.6",
-            "harparser>=0.2, <0.3",
             "pytz>=2015.07.0, <=2016.6.1",
         ]
     }

From a0932af55c9654cea31e1a14b9b548a63dc7538d Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 3 Aug 2016 12:16:47 +0530
Subject: [PATCH 04/27] Remove pages object

The HAR Spec says that the field can be left out by applications that don't
group by pages.

http://www.softwareishard.com/blog/har-12-spec/#log
---
 examples/har_dump.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 0174da538..d19cad365 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -35,7 +35,6 @@ def start():
                 "version": "0.1",
                 "comment": "mitmproxy version %s" % version.MITMPROXY
             },
-            "pages": [],
             "entries": []
         }
     })

From 99b32094d81054d7a80adb7d2a032a056465d945 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 3 Aug 2016 16:28:41 +0530
Subject: [PATCH 05/27] Add simple har_dump test

---
 test/mitmproxy/test_examples.py | 55 +++++++++++++++++++--------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index 6c24ace52..78681c4e7 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -1,16 +1,19 @@
 import json
+import os
 
 import six
-import sys
-import os.path
-from mitmproxy.flow import master
-from mitmproxy.flow import state
+
 from mitmproxy import options
 from mitmproxy import contentviews
 from mitmproxy.builtins import script
+from mitmproxy.flow import master
+from mitmproxy.flow import state
+
 import netlib.utils
+
 from netlib import tutils as netutils
 from netlib.http import Headers
+
 from . import tutils, mastertest
 
 example_dir = netlib.utils.Data(__name__).push("../../examples")
@@ -98,30 +101,34 @@ class TestScripts(mastertest.MasterTest):
         m.request(f)
         assert f.request.host == "mitmproxy.org"
 
-    def test_har_extractor(self):
-        if sys.version_info >= (3, 0):
-            with tutils.raises("does not work on Python 3"):
-                tscript("har_extractor.py")
-            return
 
+class TestHARDump(mastertest.MasterTest):
+
+    def setup(self):
+        times = dict(
+            timestamp_start=746203272,
+            timestamp_end=746203272,
+        )
+
+        # Create a dummy flow for testing
+        self.req_get = tutils.tflow(
+            req=netutils.treq(method=b'GET', content=b'', **times),
+            resp=netutils.tresp(**times)
+        )
+
+    def test_no_file_arg(self):
         with tutils.raises(ScriptError):
-            tscript("har_extractor.py")
+            tscript("har_dump.py")
 
+    def test_simple(self):
         with tutils.tmpdir() as tdir:
-            times = dict(
-                timestamp_start=746203272,
-                timestamp_end=746203272,
-            )
+            path = os.path.join(tdir, "somefile")
 
-            path = os.path.join(tdir, "file")
-            m, sc = tscript("har_extractor.py", six.moves.shlex_quote(path))
-            f = tutils.tflow(
-                req=netutils.treq(**times),
-                resp=netutils.tresp(**times)
-            )
-            m.response(f)
+            m, sc = tscript("har_dump.py", six.moves.shlex_quote(path))
+            self.invoke(m, "response", self.req_get)
             m.addons.remove(sc)
 
-            with open(path, "rb") as f:
-                test_data = json.load(f)
-            assert len(test_data["log"]["pages"]) == 1
+            with open(path, "rb") as inp:
+                har = json.load(inp)
+
+        assert len(har["log"]["entries"]) == 1

From a479c51465deae7be98ddba07ca2138fe6014d77 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Thu, 4 Aug 2016 14:16:35 +0530
Subject: [PATCH 06/27] Add method to group pairs by cookies

---
 netlib/http/cookies.py           | 31 +++++++++++++++++++++++++++++++
 test/netlib/http/test_cookies.py | 20 ++++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py
index dd0af99c3..389dbb261 100644
--- a/netlib/http/cookies.py
+++ b/netlib/http/cookies.py
@@ -26,6 +26,12 @@ variants. Serialization follows RFC6265.
     http://tools.ietf.org/html/rfc2965
 """
 
+_cookie_params = set((
+    'expires', 'path', 'comment', 'max-age',
+    'secure', 'httponly', 'version',
+))
+
+
 # TODO: Disallow LHS-only Cookie values
 
 
@@ -287,3 +293,28 @@ def is_expired(cookie_attrs):
         pass
 
     return expires or max_age
+
+
+def group_cookies(pairs):
+    """
+    Converts a list of pairs to a (name, value, attrs) for each cookie.
+    """
+
+    if not pairs:
+        return []
+
+    cookie_list = []
+
+    # First pair is always a new cookie
+    name, value = pairs[0]
+    attrs = []
+
+    for k, v in pairs[1:]:
+        if k.lower() in _cookie_params:
+            attrs.append((k, v))
+        else:
+            cookie_list.append((name, value, CookieAttrs(attrs)))
+            name, value, attrs = k, v, []
+
+    cookie_list.append((name, value, CookieAttrs(attrs)))
+    return cookie_list
diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py
index 17e21b943..5a0e264ee 100644
--- a/test/netlib/http/test_cookies.py
+++ b/test/netlib/http/test_cookies.py
@@ -266,3 +266,23 @@ def test_is_expired():
 
     assert not cookies.is_expired(CA([("Max-Age", "nan")]))
     assert not cookies.is_expired(CA([("Expires", "false")]))
+
+
+def test_group_cookies():
+    def group(cookie):
+        return cookies.group_cookies(cookies.parse_cookie_header(c))
+
+    c = "one=uno; foo=bar; foo=baz"
+    assert len(group(c)) == 3
+
+    c = "one=uno; Path=/; foo=bar; Max-Age=0; foo=baz; expires=24-08-1993"
+    assert len(group(c)) == 3
+
+    c = "one=uno;"
+    assert len(group(c)) == 1
+
+    c = "one=uno; Path=/; Max-Age=0; Expires=24-08-1993"
+    assert len(group(c)) == 1
+
+    c = "path=val; Path=/"
+    assert group(c) == [("path", "val", cookies.CookieAttrs([("Path", "/")]))]

From 55f1ffe0b100c9aa2a24b041a91091601ea4575d Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Thu, 4 Aug 2016 14:38:14 +0530
Subject: [PATCH 07/27] Format Cookies according to the HAR Spec

---
 examples/har_dump.py | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index d19cad365..390934021 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -13,6 +13,8 @@ import pytz
 import mitmproxy
 from mitmproxy import version
 
+from netlib.http import cookies
+
 HAR = {}
 
 
@@ -44,8 +46,6 @@ def response(flow):
     """
        Called when a server response has been received.
     """
-    entries = HAR["log"]["entries"]
-
     # TODO: SSL and Connect Timings
 
     # Calculate raw timings from timestamps.
@@ -78,14 +78,14 @@ def response(flow):
     response_body_decoded_size = len(flow.response.content)
     response_body_compression = response_body_decoded_size - response_body_size
 
-    entries.append({
+    HAR["log"]["entries"].append({
         "startedDateTime": started_date_time,
         "time": full_time,
         "request": {
             "method": flow.request.method,
             "url": flow.request.url,
             "httpVersion": flow.request.http_version,
-            "cookies": name_value(flow.request.cookies),
+            "cookies": format_request_cookies(flow.request.cookies.fields),
             "headers": name_value(flow.request.headers),
             "queryString": name_value(flow.request.query or {}),
             "headersSize": len(str(flow.request.headers)),
@@ -95,7 +95,7 @@ def response(flow):
             "status": flow.response.status_code,
             "statusText": flow.response.reason,
             "httpVersion": flow.response.http_version,
-            "cookies": name_value(flow.response.cookies),
+            "cookies": format_response_cookies(flow.response.cookies.fields),
             "headers": name_value(flow.response.headers),
             "content": {
                 "size": response_body_size,
@@ -127,6 +127,30 @@ def done():
     # TODO: Log results via mitmproxy.ctx.log
 
 
+def format_cookies(cookies):
+    cookie_list = []
+
+    for name, value, attrs in cookies:
+        cookie_har = {
+            "name": name,
+            "value": value,
+        }
+        cookie_har.update(attrs)
+        # print(attrs)
+
+        cookie_list.append(cookie_har)
+
+    return cookie_list
+
+
+def format_request_cookies(fields):
+    return format_cookies(cookies.group_cookies(fields))
+
+
+def format_response_cookies(fields):
+    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)
+
+
 def name_value(obj):
     """
         Convert (key, value) pairs to HAR format.

From 03e61170424bb92199cff22797135498d5ec8ce5 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 8 Aug 2016 12:55:04 +0530
Subject: [PATCH 08/27] Add a function to get cookie expiration time

---
 netlib/http/cookies.py           | 26 ++++++++++++++++++++++++++
 test/netlib/http/test_cookies.py | 20 ++++++++++++++++++++
 2 files changed, 46 insertions(+)

diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py
index 389dbb261..7f32eddff 100644
--- a/netlib/http/cookies.py
+++ b/netlib/http/cookies.py
@@ -269,6 +269,32 @@ def refresh_set_cookie_header(c, delta):
     return ret
 
 
+def get_expiration_ts(cookie_attrs):
+    """
+        Determines the time when the cookie will be expired.
+
+        Considering both 'expires' and 'max-age' parameters.
+
+        Returns: timestamp of when the cookie will expire.
+                 None, if no expiration time is set.
+    """
+    if 'expires' in cookie_attrs:
+        e = email.utils.parsedate_tz(cookie_attrs["expires"])
+        if e:
+            return email.utils.mktime_tz(e)
+
+    elif 'max-age' in cookie_attrs:
+        try:
+            max_age = int(cookie_attrs['Max-Age'])
+        except ValueError:
+            pass
+        else:
+            now_ts = time.time()
+            return now_ts + max_age
+
+    return None
+
+
 def is_expired(cookie_attrs):
     """
         Determines whether a cookie has expired.
diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py
index 5a0e264ee..cc5115c7b 100644
--- a/test/netlib/http/test_cookies.py
+++ b/test/netlib/http/test_cookies.py
@@ -1,6 +1,10 @@
+import time
+
 from netlib.http import cookies
 from netlib.tutils import raises
 
+import mock
+
 
 def test_read_token():
     tokens = [
@@ -247,6 +251,22 @@ def test_refresh_cookie():
     assert cookies.refresh_set_cookie_header(c, 0)
 
 
+@mock.patch('time.time')
+def test_get_expiration_ts(*args):
+    # Freeze time
+    now_ts = 17
+    time.time.return_value = now_ts
+
+    CA = cookies.CookieAttrs
+    F = cookies.get_expiration_ts
+
+    assert F(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT")])) == 0
+    assert F(CA([("Expires", "Thu, 24-Aug-2063 00:00:00 GMT")])) == 2955139200
+
+    assert F(CA([("Max-Age", "0")])) == now_ts
+    assert F(CA([("Max-Age", "31")])) == now_ts + 31
+
+
 def test_is_expired():
     CA = cookies.CookieAttrs
 

From 3caebe7e7369bebb44421177d9b2f4efbf0bc79b Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 8 Aug 2016 12:55:58 +0530
Subject: [PATCH 09/27] Simplify cookies.is_expired

---
 netlib/http/cookies.py | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py
index 7f32eddff..1421d8ebb 100644
--- a/netlib/http/cookies.py
+++ b/netlib/http/cookies.py
@@ -302,23 +302,14 @@ def is_expired(cookie_attrs):
         Returns: boolean
     """
 
-    # See if 'expires' time is in the past
-    expires = False
-    if 'expires' in cookie_attrs:
-        e = email.utils.parsedate_tz(cookie_attrs["expires"])
-        if e:
-            exp_ts = email.utils.mktime_tz(e)
-            now_ts = time.time()
-            expires = exp_ts < now_ts
+    exp_ts = get_expiration_ts(cookie_attrs)
+    now_ts = time.time()
 
-    # or if Max-Age is 0
-    max_age = False
-    try:
-        max_age = int(cookie_attrs.get('Max-Age', 1)) == 0
-    except ValueError:
-        pass
-
-    return expires or max_age
+    # If no expiration information was provided with the cookie
+    if exp_ts is None:
+        return False
+    else:
+        return exp_ts <= now_ts
 
 
 def group_cookies(pairs):

From a2a8283fa419ab8eb3406952802044cfabdd4f88 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 8 Aug 2016 13:56:12 +0530
Subject: [PATCH 10/27] Improve cookies formatting

---
 examples/har_dump.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 390934021..298f76a00 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -70,8 +70,7 @@ def response(flow):
     # Timings set to -1 will be ignored as per spec.
     full_time = sum(v for v in timings.values() if v > -1)
 
-    started_date_time = datetime.utcfromtimestamp(
-        flow.request.timestamp_start).replace(tzinfo=pytz.timezone("UTC")).isoformat()
+    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
 
     # Size calculations
     response_body_size = len(flow.response.content)
@@ -127,6 +126,10 @@ def done():
     # TODO: Log results via mitmproxy.ctx.log
 
 
+def format_datetime(dt):
+    return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
+
+
 def format_cookies(cookies):
     cookie_list = []
 
@@ -135,8 +138,20 @@ def format_cookies(cookies):
             "name": name,
             "value": value,
         }
-        cookie_har.update(attrs)
-        # print(attrs)
+
+        # HAR only needs some attributes
+        for key in ["path", "domain", "comment"]:
+            if key in attrs:
+                cookie_har[key] = attrs[key]
+
+        # These keys need to be boolean!
+        for key in ["httpOnly", "secure"]:
+            cookie_har[key] = bool(key in attrs)
+
+        # Expiration time needs to be formatted
+        expire_ts = cookies.get_expiration_ts(attrs)
+        if expire_ts:
+            cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
 
         cookie_list.append(cookie_har)
 

From e9c6563367c3fed05ddbdc89e35677af0f230b0d Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 8 Aug 2016 14:03:50 +0530
Subject: [PATCH 11/27] Fix wrong import

---
 examples/har_dump.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 298f76a00..b686e6921 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -11,8 +11,8 @@ from datetime import datetime
 import pytz
 
 import mitmproxy
-from mitmproxy import version
 
+from netlib import version
 from netlib.http import cookies
 
 HAR = {}

From 2c9240fd223ffaad6b6d754eda9fa1cd385e0ad1 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 8 Aug 2016 14:33:11 +0530
Subject: [PATCH 12/27] Simplify name_value

---
 examples/har_dump.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index b686e6921..819c38d8c 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -170,14 +170,4 @@ def name_value(obj):
     """
         Convert (key, value) pairs to HAR format.
     """
-
-    items = []
-    if hasattr(obj, 'fields'):
-        items = obj.fields
-    elif hasattr(obj, 'items'):
-        items = obj.items()
-
-    if items:
-        return [{"name": k, "value": v} for k, v in items]
-    else:
-        return ""
+    return [{"name": k, "value": v} for k, v in obj.items()]

From 456f80d862b1eb1a268f9780eac77c11c227ce81 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 8 Aug 2016 14:33:21 +0530
Subject: [PATCH 13/27] Open JSON file in text mode

---
 examples/har_dump.py            | 2 +-
 test/mitmproxy/test_examples.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 819c38d8c..5acd8bfad 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -120,7 +120,7 @@ def done():
         mitmproxy.ctx.log(pprint.pformat(HAR))
     # TODO: .zhar compression
     else:
-        with open(dump_file, "wb") as f:
+        with open(dump_file, "w") as f:
             f.write(json.dumps(HAR, indent=2))
 
     # TODO: Log results via mitmproxy.ctx.log
diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index 78681c4e7..62e6a652f 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -128,7 +128,7 @@ class TestHARDump(mastertest.MasterTest):
             self.invoke(m, "response", self.req_get)
             m.addons.remove(sc)
 
-            with open(path, "rb") as inp:
+            with open(path, "r") as inp:
                 har = json.load(inp)
 
         assert len(har["log"]["entries"]) == 1

From 23b9ef799eb0d322c7dcfe796b196cce3cea6435 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Tue, 9 Aug 2016 10:29:41 +0530
Subject: [PATCH 14/27] Add a ctx.log on finish

---
 examples/har_dump.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 5acd8bfad..56508cdc5 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -120,10 +120,12 @@ def done():
         mitmproxy.ctx.log(pprint.pformat(HAR))
     # TODO: .zhar compression
     else:
-        with open(dump_file, "w") as f:
-            f.write(json.dumps(HAR, indent=2))
+        json_dump = json.dumps(HAR, indent=2)
 
-    # TODO: Log results via mitmproxy.ctx.log
+        with open(dump_file, "w") as f:
+            f.write(json_dump)
+
+        mitmproxy.ctx.log("HAR log finished (wrote %s bytes to file)" % len(json_dump))
 
 
 def format_datetime(dt):

From b14eb57db190ae08266d96dbadc2458595914518 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Tue, 9 Aug 2016 11:05:16 +0530
Subject: [PATCH 15/27] Add SSL & Connect timings

---
 examples/har_dump.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 56508cdc5..d7ec90961 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -17,6 +17,10 @@ from netlib.http import cookies
 
 HAR = {}
 
+# A list of server seen till now is maintained so we can avoid
+# using 'connect' time for entries that use an existing connection.
+SERVERS_SEEN = set()
+
 
 def start():
     """
@@ -46,21 +50,33 @@ def response(flow):
     """
        Called when a server response has been received.
     """
-    # TODO: SSL and Connect Timings
 
-    # Calculate raw timings from timestamps.
+    # -1 indicates that these values do not apply to current request
+    ssl_time = -1
+    connect_time = -1
 
-    # DNS timings can not be calculated for lack of a way to measure it.
-    # The same goes for HAR blocked.
+    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
+        connect_time = (flow.server_conn.timestamp_tcp_setup -
+                        flow.server_conn.timestamp_start)
 
+        if flow.server_conn.timestamp_ssl_setup is not None:
+            ssl_time = (flow.server_conn.timestamp_ssl_setup -
+                        flow.server_conn.timestamp_tcp_setup)
+
+        SERVERS_SEEN.add(flow.server_conn)
+
+    # Calculate raw timings from timestamps. DNS timings can not be calculated
+    # for lack of a way to measure it. The same goes for HAR blocked.
     # mitmproxy will open a server connection as soon as it receives the host
     # and port from the client connection. So, the time spent waiting is actually
-    # spent waiting between request.timestamp_end and response.timestamp_start thus it
-    # correlates to HAR wait instead.
+    # spent waiting between request.timestamp_end and response.timestamp_start
+    # thus it correlates to HAR wait instead.
     timings_raw = {
         'send': flow.request.timestamp_end - flow.request.timestamp_start,
         'receive': flow.response.timestamp_end - flow.response.timestamp_start,
         'wait': flow.response.timestamp_start - flow.request.timestamp_end,
+        'connect': connect_time,
+        'ssl': ssl_time,
     }
 
     # HAR timings are integers in ms, so we re-encode the raw timings to that format.

From 7de48fc1977d02936614e18c568816717b55f889 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Tue, 9 Aug 2016 11:43:29 +0530
Subject: [PATCH 16/27] Add postData field

---
 examples/har_dump.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index d7ec90961..65cff6167 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -93,7 +93,7 @@ def response(flow):
     response_body_decoded_size = len(flow.response.content)
     response_body_compression = response_body_decoded_size - response_body_size
 
-    HAR["log"]["entries"].append({
+    entry = {
         "startedDateTime": started_date_time,
         "time": full_time,
         "request": {
@@ -123,7 +123,16 @@ def response(flow):
         },
         "cache": {},
         "timings": timings,
-    })
+    }
+
+    if flow.request.method == "POST":
+        entry["request"]["postData"] = {
+            "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
+            "text": flow.request.content,
+            "params": name_value(flow.request.urlencoded_form)
+        }
+
+    HAR["log"]["entries"].append(entry)
 
 
 def done():

From 9aa230707d4868d02df2cdc47abf4bbbcab40ba8 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Tue, 9 Aug 2016 23:43:47 +0530
Subject: [PATCH 17/27] Add serverIPAddress field

---
 examples/har_dump.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 65cff6167..6903608da 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -132,6 +132,9 @@ def response(flow):
             "params": name_value(flow.request.urlencoded_form)
         }
 
+    if flow.server_conn:
+        entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
+
     HAR["log"]["entries"].append(entry)
 
 

From ac97e5efa1f87f67fe1ac7e1b61be5f374edd65f Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 10 Aug 2016 13:23:19 +0530
Subject: [PATCH 18/27] Add text field to response content

---
 examples/har_dump.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 6903608da..bdad7766c 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -6,6 +6,7 @@ This inline script can be used to dump flows as HAR files.
 import pprint
 import json
 import sys
+import base64
 
 from datetime import datetime
 import pytz
@@ -13,6 +14,7 @@ import pytz
 import mitmproxy
 
 from netlib import version
+from netlib import strutils
 from netlib.http import cookies
 
 HAR = {}
@@ -88,8 +90,8 @@ def response(flow):
 
     started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
 
-    # Size calculations
-    response_body_size = len(flow.response.content)
+    # Response body size and encoding
+    response_body_size = len(flow.response.raw_content)
     response_body_decoded_size = len(flow.response.content)
     response_body_compression = response_body_decoded_size - response_body_size
 
@@ -125,6 +127,13 @@ def response(flow):
         "timings": timings,
     }
 
+    # Store binay data as base64
+    if strutils.is_mostly_bin(flow.response.content):
+        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content)
+        entry["response"]["content"]["encoding"] = "base64"
+    else:
+        entry["response"]["content"]["text"] = flow.response.content
+
     if flow.request.method == "POST":
         entry["request"]["postData"] = {
             "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],

From 567cbe6cb9bae3bde7b1d4d95ad70dd0db3485ff Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Wed, 10 Aug 2016 13:41:48 +0530
Subject: [PATCH 19/27] Support .zhar compression

---
 examples/har_dump.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index bdad7766c..bf8c2e15a 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -7,6 +7,7 @@ import pprint
 import json
 import sys
 import base64
+import zlib
 
 from datetime import datetime
 import pytz
@@ -155,14 +156,16 @@ def done():
 
     if dump_file == '-':
         mitmproxy.ctx.log(pprint.pformat(HAR))
-    # TODO: .zhar compression
     else:
         json_dump = json.dumps(HAR, indent=2)
 
+        if dump_file.endswith('.zhar'):
+            json_dump = zlib.compress(json_dump, 9)
+
         with open(dump_file, "w") as f:
             f.write(json_dump)
 
-        mitmproxy.ctx.log("HAR log finished (wrote %s bytes to file)" % len(json_dump))
+        mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
 
 
 def format_datetime(dt):

From 988174246b4f06ca702cbcd9aacb2e3ef7cd5280 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Thu, 11 Aug 2016 15:52:32 +0530
Subject: [PATCH 20/27] Add some cookie formatting related tests

---
 test/mitmproxy/test_examples.py  | 24 ++++++++++++++++++++++--
 test/netlib/http/test_cookies.py |  6 +++---
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index 62e6a652f..402120c05 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -13,6 +13,7 @@ import netlib.utils
 
 from netlib import tutils as netutils
 from netlib.http import Headers
+from netlib.http import cookies
 
 from . import tutils, mastertest
 
@@ -102,7 +103,7 @@ class TestScripts(mastertest.MasterTest):
         assert f.request.host == "mitmproxy.org"
 
 
-class TestHARDump(mastertest.MasterTest):
+class TestHARDump():
 
     def setup(self):
         times = dict(
@@ -125,10 +126,29 @@ class TestHARDump(mastertest.MasterTest):
             path = os.path.join(tdir, "somefile")
 
             m, sc = tscript("har_dump.py", six.moves.shlex_quote(path))
-            self.invoke(m, "response", self.req_get)
+            m.addons.invoke(m, "response", self.req_get)
             m.addons.remove(sc)
 
             with open(path, "r") as inp:
                 har = json.load(inp)
 
         assert len(har["log"]["entries"]) == 1
+
+    def test_format_cookies(self):
+        m, sc = tscript("har_dump.py", "-")
+        format_cookies = sc.ns.ns["format_cookies"]
+
+        CA = cookies.CookieAttrs
+
+        f = format_cookies([("n", "v", CA([("k", "v")]))])[0]
+        assert f['name'] == "n"
+        assert f['value'] == "v"
+        assert not f['httpOnly']
+        assert not f['secure']
+
+        f = format_cookies([("n", "v", CA([("httponly", None), ("secure", None)]))])[0]
+        assert f['httpOnly']
+        assert f['secure']
+
+        f = format_cookies([("n", "v", CA([("expires", "Fri, 24-Aug-2063 00:00:00 GMT")]))])[0]
+        assert f['expires'] == "2063-08-24T05:30:00+00:00"
diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py
index cc5115c7b..828029c6a 100644
--- a/test/netlib/http/test_cookies.py
+++ b/test/netlib/http/test_cookies.py
@@ -261,7 +261,7 @@ def test_get_expiration_ts(*args):
     F = cookies.get_expiration_ts
 
     assert F(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT")])) == 0
-    assert F(CA([("Expires", "Thu, 24-Aug-2063 00:00:00 GMT")])) == 2955139200
+    assert F(CA([("Expires", "Fri, 24-Aug-2063 00:00:00 GMT")])) == 2955139200
 
     assert F(CA([("Max-Age", "0")])) == now_ts
     assert F(CA([("Max-Age", "31")])) == now_ts + 31
@@ -280,9 +280,9 @@ def test_is_expired():
     # or both
     assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT"), ("Max-Age", "0")]))
 
-    assert not cookies.is_expired(CA([("Expires", "Thu, 24-Aug-2063 00:00:00 GMT")]))
+    assert not cookies.is_expired(CA([("Expires", "Fri, 24-Aug-2063 00:00:00 GMT")]))
     assert not cookies.is_expired(CA([("Max-Age", "1")]))
-    assert not cookies.is_expired(CA([("Expires", "Thu, 15-Jul-2068 00:00:00 GMT"), ("Max-Age", "1")]))
+    assert not cookies.is_expired(CA([("Expires", "Sun, 15-Jul-2068 00:00:00 GMT"), ("Max-Age", "1")]))
 
     assert not cookies.is_expired(CA([("Max-Age", "nan")]))
     assert not cookies.is_expired(CA([("Expires", "false")]))

From 15c488225fe3028e7f4a173de09a320f5a4c506d Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Thu, 11 Aug 2016 15:52:46 +0530
Subject: [PATCH 21/27] Refactor format_cookies

---
 examples/har_dump.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index bf8c2e15a..4ab6b5a96 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -172,10 +172,10 @@ def format_datetime(dt):
     return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
 
 
-def format_cookies(cookies):
-    cookie_list = []
+def format_cookies(cookie_list):
+    rv = []
 
-    for name, value, attrs in cookies:
+    for name, value, attrs in cookie_list:
         cookie_har = {
             "name": name,
             "value": value,
@@ -192,12 +192,12 @@ def format_cookies(cookies):
 
         # Expiration time needs to be formatted
         expire_ts = cookies.get_expiration_ts(attrs)
-        if expire_ts:
+        if expire_ts is not None:
             cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
 
-        cookie_list.append(cookie_har)
+        rv.append(cookie_har)
 
-    return cookie_list
+    return rv
 
 
 def format_request_cookies(fields):

From 4f1fb43dcce262b3ade52d93f218538c1a64e4fd Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Fri, 12 Aug 2016 01:31:33 +0530
Subject: [PATCH 22/27] Use postData field in PUT, PATCH requests too

The HAR spec isn't really clear on whether this should be the case,
but Google Chrome does this, so I think we should too.
---
 examples/har_dump.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 4ab6b5a96..803a3249e 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -135,7 +135,7 @@ def response(flow):
     else:
         entry["response"]["content"]["text"] = flow.response.content
 
-    if flow.request.method == "POST":
+    if flow.request.method in ["POST", "PUT", "PATCH"]:
         entry["request"]["postData"] = {
             "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
             "text": flow.request.content,

From bf4425de80dfa5a9b2bef2947e57a79f9655d74f Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Sun, 14 Aug 2016 13:53:18 +0530
Subject: [PATCH 23/27] Fix issue with binary content in json

---
 examples/har_dump.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/examples/har_dump.py b/examples/har_dump.py
index 803a3249e..95090edb1 100644
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -130,10 +130,11 @@ def response(flow):
 
     # Store binay data as base64
     if strutils.is_mostly_bin(flow.response.content):
-        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content)
+        b64 = base64.b64encode(flow.response.content)
+        entry["response"]["content"]["text"] = b64.decode('ascii')
         entry["response"]["content"]["encoding"] = "base64"
     else:
-        entry["response"]["content"]["text"] = flow.response.content
+        entry["response"]["content"]["text"] = flow.response.text
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
         entry["request"]["postData"] = {

From b9426fcec178395bf9c054f616ab6b23c5b2408a Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Sun, 14 Aug 2016 13:56:14 +0530
Subject: [PATCH 24/27] Add a test for base64 encoding

---
 test/mitmproxy/test_examples.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index 402120c05..99e1b34e7 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -105,16 +105,16 @@ class TestScripts(mastertest.MasterTest):
 
 class TestHARDump():
 
-    def setup(self):
+    def flow(self, resp_content=b'message'):
         times = dict(
             timestamp_start=746203272,
             timestamp_end=746203272,
         )
 
         # Create a dummy flow for testing
-        self.req_get = tutils.tflow(
-            req=netutils.treq(method=b'GET', content=b'', **times),
-            resp=netutils.tresp(**times)
+        return tutils.tflow(
+            req=netutils.treq(method=b'GET', **times),
+            resp=netutils.tresp(content=resp_content, **times)
         )
 
     def test_no_file_arg(self):
@@ -126,7 +126,7 @@ class TestHARDump():
             path = os.path.join(tdir, "somefile")
 
             m, sc = tscript("har_dump.py", six.moves.shlex_quote(path))
-            m.addons.invoke(m, "response", self.req_get)
+            m.addons.invoke(m, "response", self.flow())
             m.addons.remove(sc)
 
             with open(path, "r") as inp:
@@ -134,6 +134,19 @@ class TestHARDump():
 
         assert len(har["log"]["entries"]) == 1
 
+    def test_base64(self):
+        with tutils.tmpdir() as tdir:
+            path = os.path.join(tdir, "somefile")
+
+            m, sc = tscript("har_dump.py", six.moves.shlex_quote(path))
+            m.addons.invoke(m, "response", self.flow(resp_content=b"foo" + b"\xFF" * 10))
+            m.addons.remove(sc)
+
+            with open(path, "r") as inp:
+                har = json.load(inp)
+
+        assert har["log"]["entries"][0]["response"]["content"]["encoding"] == "base64"
+
     def test_format_cookies(self):
         m, sc = tscript("har_dump.py", "-")
         format_cookies = sc.ns.ns["format_cookies"]

From ec0bae35c538d8b4fde1c5b46b162c19783ca1a7 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 15 Aug 2016 10:39:38 +0530
Subject: [PATCH 25/27] Assert cookie groups explicitly rather than just the
 length

---
 test/netlib/http/test_cookies.py | 56 +++++++++++++++++++++++---------
 1 file changed, 40 insertions(+), 16 deletions(-)

diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py
index 828029c6a..56e0b21b9 100644
--- a/test/netlib/http/test_cookies.py
+++ b/test/netlib/http/test_cookies.py
@@ -289,20 +289,44 @@ def test_is_expired():
 
 
 def test_group_cookies():
-    def group(cookie):
-        return cookies.group_cookies(cookies.parse_cookie_header(c))
+    CA = cookies.CookieAttrs
+    groups = [
+        [
+            "one=uno; foo=bar; foo=baz",
+            [
+                ('one', 'uno', CA([])),
+                ('foo', 'bar', CA([])),
+                ('foo', 'baz', CA([]))
+            ]
+        ],
+        [
+            "one=uno; Path=/; foo=bar; Max-Age=0; foo=baz; expires=24-08-1993",
+            [
+                ('one', 'uno', CA([('Path', '/')])),
+                ('foo', 'bar', CA([('Max-Age', '0')])),
+                ('foo', 'baz', CA([('expires', '24-08-1993')]))
+            ]
+        ],
+        [
+            "one=uno;",
+            [
+                ('one', 'uno', CA([]))
+            ]
+        ],
+        [
+            "one=uno; Path=/; Max-Age=0; Expires=24-08-1993",
+            [
+                ('one', 'uno', CA([('Path', '/'), ('Max-Age', '0'), ('Expires', '24-08-1993')]))
+            ]
+        ],
+        [
+            "path=val; Path=/",
+            [
+                ('path', 'val', CA([('Path', '/')]))
+            ]
+        ]
+    ]
 
-    c = "one=uno; foo=bar; foo=baz"
-    assert len(group(c)) == 3
-
-    c = "one=uno; Path=/; foo=bar; Max-Age=0; foo=baz; expires=24-08-1993"
-    assert len(group(c)) == 3
-
-    c = "one=uno;"
-    assert len(group(c)) == 1
-
-    c = "one=uno; Path=/; Max-Age=0; Expires=24-08-1993"
-    assert len(group(c)) == 1
-
-    c = "path=val; Path=/"
-    assert group(c) == [("path", "val", cookies.CookieAttrs([("Path", "/")]))]
+    for c, expected in groups:
+        observed = cookies.group_cookies(cookies.parse_cookie_header(c))
+        assert observed == expected

From bc175b58f247b412fdcdbe2297d26b0151db8285 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 15 Aug 2016 10:46:42 +0530
Subject: [PATCH 26/27] Use test times less than 2038

So that tests pass on 32bit systems too
---
 test/mitmproxy/test_examples.py  | 4 ++--
 test/netlib/http/test_cookies.py | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index 99e1b34e7..e03d34e96 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -163,5 +163,5 @@ class TestHARDump():
         assert f['httpOnly']
         assert f['secure']
 
-        f = format_cookies([("n", "v", CA([("expires", "Fri, 24-Aug-2063 00:00:00 GMT")]))])[0]
-        assert f['expires'] == "2063-08-24T05:30:00+00:00"
+        f = format_cookies([("n", "v", CA([("expires", "Mon, 24-Aug-2037 00:00:00 GMT")]))])[0]
+        assert f['expires'] == "2037-08-24T05:30:00+00:00"
diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py
index 56e0b21b9..efd8ba809 100644
--- a/test/netlib/http/test_cookies.py
+++ b/test/netlib/http/test_cookies.py
@@ -261,7 +261,7 @@ def test_get_expiration_ts(*args):
     F = cookies.get_expiration_ts
 
     assert F(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT")])) == 0
-    assert F(CA([("Expires", "Fri, 24-Aug-2063 00:00:00 GMT")])) == 2955139200
+    assert F(CA([("Expires", "Mon, 24-Aug-2037 00:00:00 GMT")])) == 2134684800
 
     assert F(CA([("Max-Age", "0")])) == now_ts
     assert F(CA([("Max-Age", "31")])) == now_ts + 31
@@ -280,9 +280,9 @@ def test_is_expired():
     # or both
     assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT"), ("Max-Age", "0")]))
 
-    assert not cookies.is_expired(CA([("Expires", "Fri, 24-Aug-2063 00:00:00 GMT")]))
+    assert not cookies.is_expired(CA([("Expires", "Mon, 24-Aug-2037 00:00:00 GMT")]))
     assert not cookies.is_expired(CA([("Max-Age", "1")]))
-    assert not cookies.is_expired(CA([("Expires", "Sun, 15-Jul-2068 00:00:00 GMT"), ("Max-Age", "1")]))
+    assert not cookies.is_expired(CA([("Expires", "Wed, 15-Jul-2037 00:00:00 GMT"), ("Max-Age", "1")]))
 
     assert not cookies.is_expired(CA([("Max-Age", "nan")]))
     assert not cookies.is_expired(CA([("Expires", "false")]))

From 239eaf0078db0549232ef6eca54dabf75e9752c5 Mon Sep 17 00:00:00 2001
From: Shadab Zafar <dufferzafar0@gmail.com>
Date: Mon, 15 Aug 2016 14:28:13 +0530
Subject: [PATCH 27/27] Remove timezone specific value

---
 test/mitmproxy/test_examples.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index e03d34e96..83a37a36c 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -164,4 +164,4 @@ class TestHARDump():
         assert f['secure']
 
         f = format_cookies([("n", "v", CA([("expires", "Mon, 24-Aug-2037 00:00:00 GMT")]))])[0]
-        assert f['expires'] == "2037-08-24T05:30:00+00:00"
+        assert f['expires']