Utils cleanups

- Move more stuff that belongs in netlib.human
- Move some stuff to near the only use
- Zap mitmproxy.utils.timestamp(). I see the rationale, but we used it
interchangeably with time.time() throughout the project. Since time.time()
dominates in the codebase and timestamp() is such low utility, away it goes.

mitmproxy/console/common.py

@@ -136,7 +136,7 @@ def raw_format_flow(f, focus, extended):
     if extended:
         req.append(
             fcol(
-                utils.format_timestamp(f["req_timestamp"]),
+                human.format_timestamp(f["req_timestamp"]),
                 "highlight"
             )
         )

mitmproxy/console/flowdetailview.py

@@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division
 
 import urwid
 
-from mitmproxy import utils
 from mitmproxy.console import common, searchable
+from netlib import human
 
 
 def maybe_timestamp(base, attr):
     if base is not None and getattr(base, attr):
-        return utils.format_timestamp_with_milli(getattr(base, attr))
+        return human.format_timestamp_with_milli(getattr(base, attr))
     else:
         return "active"
 

mitmproxy/contentviews.py

@@ -28,7 +28,6 @@ from PIL import ExifTags
 from PIL import Image
 from six.moves import cStringIO as StringIO
 
-import mitmproxy.utils
 from mitmproxy import exceptions
 from mitmproxy.contrib import jsbeautifier
 from mitmproxy.contrib.wbxml import ASCommandResponse
@@ -62,6 +61,14 @@ VIEW_CUTOFF = 512
 KEY_MAX = 30
 
 
+def pretty_json(s):
+    try:
+        p = json.loads(s)
+    except ValueError:
+        return None
+    return json.dumps(p, sort_keys=True, indent=4)
+
+
 def format_dict(d):
     """
     Helper function that transforms the given dictionary into a list of
@@ -215,9 +222,9 @@ class ViewJSON(View):
     content_types = ["application/json"]
 
     def __call__(self, data, **metadata):
-        pretty_json = mitmproxy.utils.pretty_json(data)
+        pj = pretty_json(data)
-        if pretty_json:
+        if pj:
-            return "JSON", format_text(pretty_json)
+            return "JSON", format_text(pj)
 
 
 class ViewHTML(View):

mitmproxy/models/connections.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import, print_function, division
+import time
 
 import copy
 import os
@@ -6,7 +7,6 @@ import os
 import six
 
 from mitmproxy import stateobject
-from mitmproxy import utils
 from netlib import certutils
 from netlib import tcp
 
@@ -39,7 +39,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
             self.clientcert = None
             self.ssl_established = None
 
-        self.timestamp_start = utils.timestamp()
+        self.timestamp_start = time.time()
         self.timestamp_end = None
         self.timestamp_ssl_setup = None
         self.protocol = None
@@ -97,11 +97,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
 
     def convert_to_ssl(self, *args, **kwargs):
         super(ClientConnection, self).convert_to_ssl(*args, **kwargs)
-        self.timestamp_ssl_setup = utils.timestamp()
+        self.timestamp_ssl_setup = time.time()
 
     def finish(self):
         super(ClientConnection, self).finish()
-        self.timestamp_end = utils.timestamp()
+        self.timestamp_end = time.time()
 
 
 class ServerConnection(tcp.TCPClient, stateobject.StateObject):
@@ -194,9 +194,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         return copy.copy(self)
 
     def connect(self):
-        self.timestamp_start = utils.timestamp()
+        self.timestamp_start = time.time()
         tcp.TCPClient.connect(self)
-        self.timestamp_tcp_setup = utils.timestamp()
+        self.timestamp_tcp_setup = time.time()
 
     def send(self, message):
         if isinstance(message, list):
@@ -218,11 +218,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
 
         self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs)
         self.sni = sni
-        self.timestamp_ssl_setup = utils.timestamp()
+        self.timestamp_ssl_setup = time.time()
 
     def finish(self):
         tcp.TCPClient.finish(self)
-        self.timestamp_end = utils.timestamp()
+        self.timestamp_end = time.time()
 
 
 ServerConnection._stateobject_attributes["via"] = ServerConnection

mitmproxy/models/flow.py

@@ -1,11 +1,11 @@
 from __future__ import absolute_import, print_function, division
 
+import time
 import copy
 import uuid
 
 from mitmproxy import exceptions
 from mitmproxy import stateobject
-from mitmproxy import utils
 from mitmproxy import version
 from mitmproxy.models.connections import ClientConnection
 from mitmproxy.models.connections import ServerConnection
@@ -34,7 +34,7 @@ class Error(stateobject.StateObject):
         @type timestamp: float
         """
         self.msg = msg
-        self.timestamp = timestamp or utils.timestamp()
+        self.timestamp = timestamp or time.time()
 
     _stateobject_attributes = dict(
         msg=str,

mitmproxy/protocol/http.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import, print_function, division
 
+import time
 import sys
 import traceback
 
@@ -9,7 +10,6 @@ import six
 import netlib.exceptions
 from mitmproxy import exceptions
 from mitmproxy import models
-from mitmproxy import utils
 from mitmproxy.protocol import base
 from netlib import http
 from netlib import tcp
@@ -265,7 +265,7 @@ class HttpLayer(base.Layer):
             if callable(flow.response.stream):
                 chunks = flow.response.stream(chunks)
             self.send_response_body(flow.response, chunks)
-            flow.response.timestamp_end = utils.timestamp()
+            flow.response.timestamp_end = time.time()
 
     def get_response_from_server(self, flow):
         def get_response():
@@ -310,7 +310,7 @@ class HttpLayer(base.Layer):
                 flow.request,
                 flow.response
             ))
-        flow.response.timestamp_end = utils.timestamp()
+        flow.response.timestamp_end = time.time()
 
         # no further manipulation of self.server_conn beyond this point
         # we can safely set it as the final attribute value here.

mitmproxy/utils.py

@@ -1,38 +1,8 @@
 from __future__ import absolute_import, print_function, division
 
-import datetime
-import json
-import time
-
 import netlib.utils
 
 
-def timestamp():
-    """
-        Returns a serializable UTC timestamp.
-    """
-    return time.time()
-
-
-def format_timestamp(s):
-    s = time.localtime(s)
-    d = datetime.datetime.fromtimestamp(time.mktime(s))
-    return d.strftime("%Y-%m-%d %H:%M:%S")
-
-
-def format_timestamp_with_milli(s):
-    d = datetime.datetime.fromtimestamp(s)
-    return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
-
-
-def pretty_json(s):
-    try:
-        p = json.loads(s)
-    except ValueError:
-        return None
-    return json.dumps(p, sort_keys=True, indent=4)
-
-
 pkg_data = netlib.utils.Data(__name__)
 
 

netlib/http/http1/assemble.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, print_function, division
 
-from netlib import utils
+import netlib.http.url
 from netlib import exceptions
 
 
@@ -82,7 +82,7 @@ def _assemble_request_headers(request_data):
     """
     headers = request_data.headers.copy()
     if "host" not in headers and request_data.scheme and request_data.host and request_data.port:
-        headers["host"] = utils.hostport(
+        headers["host"] = netlib.http.url.hostport(
             request_data.scheme,
             request_data.host,
             request_data.port

netlib/http/url.py

@@ -78,7 +78,7 @@ def unparse(scheme, host, port, path=""):
     """
     if path == "*":
         path = ""
-    return "%s://%s%s" % (scheme, utils.hostport(scheme, host, port), path)
+    return "%s://%s%s" % (scheme, hostport(scheme, host, port), path)
 
 
 def encode(s):
@@ -94,3 +94,16 @@ def decode(s):
         Takes a urlencoded string and returns a list of (key, value) tuples.
     """
     return urllib.parse.parse_qsl(s, keep_blank_values=True)
+
+
+def hostport(scheme, host, port):
+    """
+        Returns the host component, with a port specifcation if needed.
+    """
+    if (port, scheme) in [(80, "http"), (443, "https"), (80, b"http"), (443, b"https")]:
+        return host
+    else:
+        if isinstance(host, six.binary_type):
+            return b"%s:%d" % (host, port)
+        else:
+            return "%s:%d" % (host, port)

netlib/human.py

@@ -1,3 +1,6 @@
+import datetime
+import time
+
 
 SIZE_TABLE = [
     ("b", 1024 ** 0),
@@ -48,3 +51,14 @@ def pretty_duration(secs):
             return formatter.format(secs)
     # less than 1 sec
     return "{:.0f}ms".format(secs * 1000)
+
+
+def format_timestamp(s):
+    s = time.localtime(s)
+    d = datetime.datetime.fromtimestamp(time.mktime(s))
+    return d.strftime("%Y-%m-%d %H:%M:%S")
+
+
+def format_timestamp_with_milli(s):
+    d = datetime.datetime.fromtimestamp(s)
+    return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]

netlib/utils.py

@@ -4,8 +4,6 @@ import re
 import importlib
 import inspect
 
-import six
-
 
 def setbit(byte, offset, value):
     """
@@ -94,16 +92,3 @@ def is_valid_host(host):
 
 def is_valid_port(port):
     return 0 <= port <= 65535
-
-
-def hostport(scheme, host, port):
-    """
-        Returns the host component, with a port specifcation if needed.
-    """
-    if (port, scheme) in [(80, "http"), (443, "https"), (80, b"http"), (443, b"https")]:
-        return host
-    else:
-        if isinstance(host, six.binary_type):
-            return b"%s:%d" % (host, port)
-        else:
-            return "%s:%d" % (host, port)

pathod/pathod_cmdline.py

@@ -5,7 +5,16 @@ import os.path
 import re
 
 from netlib import tcp, human
-from . import pathod, version, utils
+from . import pathod, version
+
+
+def parse_anchor_spec(s):
+    """
+        Return a tuple, or None on error.
+    """
+    if "=" not in s:
+        return None
+    return tuple(s.split("=", 1))
 
 
 def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
@@ -188,7 +197,7 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
 
     alst = []
     for i in args.anchors:
-        parts = utils.parse_anchor_spec(i)
+        parts = parse_anchor_spec(i)
         if not parts:
             return parser.error("Invalid anchor specification: %s" % i)
         alst.append(parts)

pathod/utils.py

@@ -17,15 +17,6 @@ class MemBool(object):
         return bool(v)
 
 
-def parse_anchor_spec(s):
-    """
-        Return a tuple, or None on error.
-    """
-    if "=" not in s:
-        return None
-    return tuple(s.split("=", 1))
-
-
 data = netlib.utils.Data(__name__)
 
 

test/mitmproxy/test_contentview.py

@@ -1,3 +1,5 @@
+import json
+
 from mitmproxy.exceptions import ContentViewException
 from netlib.http import Headers
 from netlib.odict import ODict
@@ -274,3 +276,9 @@ if cv.ViewProtobuf.is_available():
 
 def test_get_by_shortcut():
     assert cv.get_by_shortcut("h")
+
+
+def test_pretty_json():
+    s = json.dumps({"foo": 1})
+    assert cv.pretty_json(s)
+    assert not cv.pretty_json("moo")

test/mitmproxy/test_utils.py

@@ -1,29 +1,14 @@
-import json
 from mitmproxy import utils
 from . import tutils
 
 utils.CERT_SLEEP_TIME = 0
 
 
-def test_format_timestamp():
-    assert utils.format_timestamp(utils.timestamp())
-
-
-def test_format_timestamp_with_milli():
-    assert utils.format_timestamp_with_milli(utils.timestamp())
-
-
 def test_pkg_data():
     assert utils.pkg_data.path("console")
     tutils.raises("does not exist", utils.pkg_data.path, "nonexistent")
 
 
-def test_pretty_json():
-    s = json.dumps({"foo": 1})
-    assert utils.pretty_json(s)
-    assert not utils.pretty_json("moo")
-
-
 def test_LRUCache():
     cache = utils.LRUCache(2)
 

test/netlib/test_human.py

@@ -1,6 +1,15 @@
+import time
 from netlib import human, tutils
 
 
+def test_format_timestamp():
+    assert human.format_timestamp(time.time())
+
+
+def test_format_timestamp_with_milli():
+    assert human.format_timestamp_with_milli(time.time())
+
+
 def test_parse_size():
     assert human.parse_size("0") == 0
     assert human.parse_size("0b") == 0

test/pathod/test_pathod_cmdline.py

@@ -3,6 +3,11 @@ import tutils
 import mock
 
 
+def test_parse_anchor_spec():
+    assert cmdline.parse_anchor_spec("foo=200") == ("foo", "200")
+    assert cmdline.parse_anchor_spec("foo") is None
+
+
 @mock.patch("argparse.ArgumentParser.error")
 def test_pathod(perror):
     assert cmdline.args_pathod(["pathod"])

test/pathod/test_utils.py

@@ -11,10 +11,5 @@ def test_membool():
     assert m.v == 2
 
 
-def test_parse_anchor_spec():
-    assert utils.parse_anchor_spec("foo=200") == ("foo", "200")
-    assert utils.parse_anchor_spec("foo") is None
-
-
 def test_data_path():
     tutils.raises(ValueError, utils.data.path, "nonexistent")