The final piece: netlib -> mitproxy.net

.appveyor.yml

@@ -23,7 +23,7 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod -v"
+  - ps: "tox -- --cov mitmproxy --cov pathod -v"
 
 deploy_script:
   ps: |

.travis.yml

@@ -49,7 +49,7 @@ install:
     fi
   - pip install tox
 
-script: tox -- --cov netlib --cov mitmproxy --cov pathod -v
+script: tox -- --cov mitmproxy --cov pathod -v
 
 after_success:
   - |

MANIFEST.in

@@ -1,4 +1,3 @@
 graft mitmproxy
 graft pathod
-graft netlib
+recursive-exclude * *.pyc *.pyo *.swo *.swp *.map
-recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

README.rst

@@ -3,8 +3,7 @@ mitmproxy
 
 |travis| |appveyor| |coverage| |latest_release| |python_versions|
 
-This repository contains the **mitmproxy** and **pathod** projects, as well as
+This repository contains the **mitmproxy** and **pathod** projects.
-their shared networking library, **netlib**.
 
 ``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console
 interface.
@@ -64,7 +63,7 @@ virtualenv_ installed (you can find installation instructions for virtualenv
 
 The *dev* script will create a virtualenv environment in a directory called
 "venv", and install all mandatory and optional dependencies into it. The
-primary mitmproxy components - mitmproxy, netlib and pathod - are installed as
+primary mitmproxy components - mitmproxy and pathod - are installed as
 "editable", so any changes to the source in the repository will be reflected
 live in the virtualenv.
 
@@ -144,7 +143,7 @@ PR checks will fail and block merging. We are using this command to check for st
 
 .. code-block:: text
 
-    flake8 --jobs 8 --count mitmproxy netlib pathod examples test
+    flake8 --jobs 8 --count mitmproxy pathod examples test
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg

docs/conf.py

@@ -231,10 +231,7 @@ def linkcode_resolve(domain, info):
         _, line = inspect.getsourcelines(obj)
     except (TypeError, IOError):
         return None
-    if spath.rfind("netlib") > -1:
+    if spath.rfind("mitmproxy") > -1:
-        off = spath.rfind("netlib")
-        mpath = spath[off:]
-    elif spath.rfind("mitmproxy") > -1:
         off = spath.rfind("mitmproxy")
         mpath = spath[off:]
     else:

docs/dev/testing.rst

@@ -10,7 +10,7 @@ suitable extension to the test suite.
 Our tests are written for the `py.test`_ or nose_ test frameworks.
 At the point where you send your pull request, a command like this:
 
->>> py.test --cov mitmproxy --cov netlib
+>>> py.test --cov mitmproxy
 
 Should give output something like this:
 

docs/modd.conf

@@ -1,6 +1,6 @@
 @build = ./_build
 
-** !_build/** ../netlib/**/*.py ../mitmproxy/**/*.py {
+** !_build/** ../mitmproxy/**/*.py {
     prep: sphinx-build -W -d @build/doctrees -b html . @build/html
     daemon: devd -m @build/html
 }

examples/har_dump.py

@@ -16,7 +16,7 @@ import mitmproxy
 
 from mitmproxy import version
 from mitmproxy.utils import strutils
-from netlib.http import cookies
+from mitmproxy.net.http import cookies
 
 HAR = {}
 

mitmproxy/addons/stickycookie.py

@@ -1,7 +1,7 @@
 import collections
 from http import cookiejar
 
-from netlib.http import cookies
+from mitmproxy.net.http import cookies
 
 from mitmproxy import exceptions
 from mitmproxy import flowfilter

mitmproxy/addons/streambodies.py

@@ -1,4 +1,4 @@
-from netlib.http import http1
+from mitmproxy.net.http import http1
 from mitmproxy import exceptions
 from mitmproxy import ctx
 

mitmproxy/addons/wsgiapp.py

@@ -1,7 +1,7 @@
 from mitmproxy import ctx
 from mitmproxy import exceptions
 
-from netlib import wsgi
+from mitmproxy.net import wsgi
 from mitmproxy import version
 
 

mitmproxy/certs.py

@@ -193,7 +193,7 @@ class CertStore:
     @staticmethod
     def load_dhparam(path):
 
-        # netlib<=0.10 doesn't generate a dhparam file.
+        # mitmproxy<=0.10 doesn't generate a dhparam file.
         # Create it now if neccessary.
         if not os.path.exists(path):
             with open(path, "wb") as f:

mitmproxy/connections.py

@@ -5,7 +5,7 @@ import os
 
 from mitmproxy import stateobject
 from mitmproxy import certs
-from netlib import tcp
+from mitmproxy.net import tcp
 
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):

mitmproxy/contentviews.py

@@ -33,10 +33,10 @@ from PIL import ExifTags
 from PIL import Image
 from mitmproxy import exceptions
 from mitmproxy.contrib.wbxml import ASCommandResponse
-from netlib import http
+from mitmproxy.net import http
 from mitmproxy.types import multidict
 from mitmproxy.utils import strutils
-from netlib.http import url
+from mitmproxy.net.http import url
 
 try:
     import pyamf

mitmproxy/exceptions.py

@@ -112,7 +112,7 @@ class AddonHalt(MitmproxyException):
 
 class NetlibException(MitmproxyException):
     """
-    Base class for all exceptions thrown by netlib.
+    Base class for all exceptions thrown by mitmproxy.net.
     """
     def __init__(self, message=None):
         super().__init__(message)

mitmproxy/export.py

@@ -3,7 +3,7 @@ import re
 import textwrap
 import urllib
 
-import netlib.http
+import mitmproxy.net.http
 
 
 def _native(s):
@@ -89,9 +89,9 @@ def python_code(flow):
     return code
 
 
-def is_json(headers: netlib.http.Headers, content: bytes) -> bool:
+def is_json(headers: mitmproxy.net.http.Headers, content: bytes) -> bool:
     if headers:
-        ct = netlib.http.parse_content_type(headers.get("content-type", ""))
+        ct = mitmproxy.net.http.parse_content_type(headers.get("content-type", ""))
         if ct and "%s/%s" % (ct[0], ct[1]) == "application/json":
             try:
                 return json.loads(content.decode("utf8", "surrogateescape"))

mitmproxy/http.py

@@ -1,9 +1,9 @@
 import cgi
 
 from mitmproxy import flow
-from netlib import http
+from mitmproxy.net import http
 from mitmproxy import version
-from netlib import tcp
+from mitmproxy.net import tcp
 
 
 class HTTPRequest(http.Request):
@@ -12,7 +12,7 @@ class HTTPRequest(http.Request):
     A mitmproxy HTTP request.
     """
 
-    # This is a very thin wrapper on top of :py:class:`netlib.http.Request` and
+    # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Request` and
     # may be removed in the future.
 
     def __init__(
@@ -73,7 +73,7 @@ class HTTPRequest(http.Request):
     @classmethod
     def wrap(self, request):
         """
-        Wraps an existing :py:class:`netlib.http.Request`.
+        Wraps an existing :py:class:`mitmproxy.net.http.Request`.
         """
         req = HTTPRequest(
             first_line_format=request.data.first_line_format,
@@ -99,7 +99,7 @@ class HTTPResponse(http.Response):
     """
     A mitmproxy HTTP response.
     """
-    # This is a very thin wrapper on top of :py:class:`netlib.http.Response` and
+    # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Response` and
     # may be removed in the future.
 
     def __init__(
@@ -131,7 +131,7 @@ class HTTPResponse(http.Response):
     @classmethod
     def wrap(self, response):
         """
-        Wraps an existing :py:class:`netlib.http.Response`.
+        Wraps an existing :py:class:`mitmproxy.net.http.Response`.
         """
         resp = HTTPResponse(
             http_version=response.data.http_version,

mitmproxy/master.py

@@ -15,7 +15,7 @@ from mitmproxy import log
 from mitmproxy import io
 from mitmproxy.proxy.protocol import http_replay
 from mitmproxy.types import basethread
-import netlib.http
+import mitmproxy.net.http
 
 from . import ctx as mitmproxy_ctx
 
@@ -122,7 +122,7 @@ class Master:
         s = connections.ServerConnection.make_dummy((host, port))
 
         f = http.HTTPFlow(c, s)
-        headers = netlib.http.Headers()
+        headers = mitmproxy.net.http.Headers()
 
         req = http.HTTPRequest(
             "absolute",

mitmproxy/net/http/__init__.py

@@ -0,0 +1,15 @@
+from mitmproxy.net.http.request import Request
+from mitmproxy.net.http.response import Response
+from mitmproxy.net.http.message import Message
+from mitmproxy.net.http.headers import Headers, parse_content_type
+from mitmproxy.net.http.message import decoded
+from mitmproxy.net.http import http1, http2, status_codes, multipart
+
+__all__ = [
+    "Request",
+    "Response",
+    "Message",
+    "Headers", "parse_content_type",
+    "decoded",
+    "http1", "http2", "status_codes", "multipart",
+]

mitmproxy/net/http/http1/assemble.py

@@ -1,4 +1,4 @@
-import netlib.http.url
+import mitmproxy.net.http.url
 from mitmproxy import exceptions
 
 
@@ -44,7 +44,7 @@ def assemble_body(headers, body_chunks):
 def _assemble_request_line(request_data):
     """
     Args:
-        request_data (netlib.http.request.RequestData)
+        request_data (mitmproxy.net.http.request.RequestData)
     """
     form = request_data.first_line_format
     if form == "relative":
@@ -76,11 +76,11 @@ def _assemble_request_line(request_data):
 def _assemble_request_headers(request_data):
     """
     Args:
-        request_data (netlib.http.request.RequestData)
+        request_data (mitmproxy.net.http.request.RequestData)
     """
     headers = request_data.headers.copy()
     if "host" not in headers and request_data.scheme and request_data.host and request_data.port:
-        headers["host"] = netlib.http.url.hostport(
+        headers["host"] = mitmproxy.net.http.url.hostport(
             request_data.scheme,
             request_data.host,
             request_data.port

mitmproxy/net/http/http1/read.py

@@ -2,11 +2,11 @@ import time
 import sys
 import re
 
-from netlib.http import request
+from mitmproxy.net.http import request
-from netlib.http import response
+from mitmproxy.net.http import response
-from netlib.http import headers
+from mitmproxy.net.http import headers
-from netlib.http import url
+from mitmproxy.net.http import url
-from netlib import check
+from mitmproxy.net import check
 from mitmproxy import exceptions
 
 

mitmproxy/net/http/http2/__init__.py

@@ -0,0 +1,8 @@
+from mitmproxy.net.http.http2.framereader import read_raw_frame, parse_frame
+from mitmproxy.net.http.http2.utils import parse_headers
+
+__all__ = [
+    "read_raw_frame",
+    "parse_frame",
+    "parse_headers",
+]

mitmproxy/net/http/http2/utils.py

@@ -1,4 +1,4 @@
-from netlib.http import url
+from mitmproxy.net.http import url
 
 
 def parse_headers(headers):

mitmproxy/net/http/message.py

@@ -3,9 +3,9 @@ import warnings
 from typing import Optional
 
 from mitmproxy.utils import strutils
-from netlib.http import encoding
+from mitmproxy.net.http import encoding
 from mitmproxy.types import serializable
-from netlib.http import headers
+from mitmproxy.net.http import headers
 
 
 # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
@@ -69,7 +69,7 @@ class Message(serializable.Serializable):
         Message headers object
 
         Returns:
-            netlib.http.Headers
+            mitmproxy.net.http.Headers
         """
         return self.data.headers
 

mitmproxy/net/http/multipart.py

@@ -1,6 +1,6 @@
 import re
 
-from netlib.http import headers
+from mitmproxy.net.http import headers
 
 
 def decode(hdrs, content):

mitmproxy/net/http/request.py

@@ -3,11 +3,11 @@ import urllib
 
 from mitmproxy.types import multidict
 from mitmproxy.utils import strutils
-from netlib.http import multipart
+from mitmproxy.net.http import multipart
-from netlib.http import cookies
+from mitmproxy.net.http import cookies
-from netlib.http import headers as nheaders
+from mitmproxy.net.http import headers as nheaders
-from netlib.http import message
+from mitmproxy.net.http import message
-import netlib.http.url
+import mitmproxy.net.http.url
 
 # This regex extracts & splits the host header into host and port.
 # Handles the edge case of IPv6 addresses containing colons.
@@ -203,11 +203,11 @@ class Request(message.Message):
         """
         if self.first_line_format == "authority":
             return "%s:%d" % (self.host, self.port)
-        return netlib.http.url.unparse(self.scheme, self.host, self.port, self.path)
+        return mitmproxy.net.http.url.unparse(self.scheme, self.host, self.port, self.path)
 
     @url.setter
     def url(self, url):
-        self.scheme, self.host, self.port, self.path = netlib.http.url.parse(url)
+        self.scheme, self.host, self.port, self.path = mitmproxy.net.http.url.parse(url)
 
     def _parse_host_header(self):
         """Extract the host and port from Host header"""
@@ -243,12 +243,12 @@ class Request(message.Message):
         """
         if self.first_line_format == "authority":
             return "%s:%d" % (self.pretty_host, self.port)
-        return netlib.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path)
+        return mitmproxy.net.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path)
 
     @property
     def query(self) -> multidict.MultiDictView:
         """
-        The request query string as an :py:class:`~netlib.multidict.MultiDictView` object.
+        The request query string as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
         """
         return multidict.MultiDictView(
             self._get_query,
@@ -257,10 +257,10 @@ class Request(message.Message):
 
     def _get_query(self):
         query = urllib.parse.urlparse(self.url).query
-        return tuple(netlib.http.url.decode(query))
+        return tuple(mitmproxy.net.http.url.decode(query))
 
     def _set_query(self, query_data):
-        query = netlib.http.url.encode(query_data)
+        query = mitmproxy.net.http.url.encode(query_data)
         _, _, path, params, _, fragment = urllib.parse.urlparse(self.url)
         self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
 
@@ -273,7 +273,7 @@ class Request(message.Message):
         """
         The request cookies.
 
-        An empty :py:class:`~netlib.multidict.MultiDictView` object if the cookie monster ate them all.
+        An empty :py:class:`~mitmproxy.net.multidict.MultiDictView` object if the cookie monster ate them all.
         """
         return multidict.MultiDictView(
             self._get_cookies,
@@ -301,11 +301,11 @@ class Request(message.Message):
         # This needs to be a tuple so that it's immutable.
         # Otherwise, this would fail silently:
         #   request.path_components.append("foo")
-        return tuple(netlib.http.url.unquote(i) for i in path.split("/") if i)
+        return tuple(mitmproxy.net.http.url.unquote(i) for i in path.split("/") if i)
 
     @path_components.setter
     def path_components(self, components):
-        components = map(lambda x: netlib.http.url.quote(x, safe=""), components)
+        components = map(lambda x: mitmproxy.net.http.url.quote(x, safe=""), components)
         path = "/" + "/".join(components)
         _, _, _, params, query, fragment = urllib.parse.urlparse(self.url)
         self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
@@ -347,7 +347,7 @@ class Request(message.Message):
     @property
     def urlencoded_form(self):
         """
-        The URL-encoded form data as an :py:class:`~netlib.multidict.MultiDictView` object.
+        The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
         An empty multidict.MultiDictView if the content-type indicates non-form data
         or the content could not be parsed.
         """
@@ -360,7 +360,7 @@ class Request(message.Message):
         is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
             try:
-                return tuple(netlib.http.url.decode(self.content))
+                return tuple(mitmproxy.net.http.url.decode(self.content))
             except ValueError:
                 pass
         return ()
@@ -371,7 +371,7 @@ class Request(message.Message):
         This will overwrite the existing content if there is one.
         """
         self.headers["content-type"] = "application/x-www-form-urlencoded"
-        self.content = netlib.http.url.encode(form_data).encode()
+        self.content = mitmproxy.net.http.url.encode(form_data).encode()
 
     @urlencoded_form.setter
     def urlencoded_form(self, value):
@@ -380,7 +380,7 @@ class Request(message.Message):
     @property
     def multipart_form(self):
         """
-        The multipart form data as an :py:class:`~netlib.multidict.MultiDictView` object.
+        The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
         None if the content-type indicates non-form data.
         """
         return multidict.MultiDictView(

mitmproxy/net/http/response.py

@@ -2,10 +2,10 @@ import time
 from email.utils import parsedate_tz, formatdate, mktime_tz
 from mitmproxy.utils import human
 from mitmproxy.types import multidict
-from netlib.http import cookies
+from mitmproxy.net.http import cookies
-from netlib.http import headers as nheaders
+from mitmproxy.net.http import headers as nheaders
-from netlib.http import message
+from mitmproxy.net.http import message
-from netlib.http import status_codes
+from mitmproxy.net.http import status_codes
 from typing import AnyStr
 from typing import Dict
 from typing import Iterable
@@ -131,7 +131,7 @@ class Response(message.Message):
     def cookies(self) -> multidict.MultiDictView:
         """
         The response cookies. A possibly empty
-        :py:class:`~netlib.multidict.MultiDictView`, where the keys are cookie
+        :py:class:`~mitmproxy.net.multidict.MultiDictView`, where the keys are cookie
         name strings, and values are (value, attr) tuples. Value is a string,
         and attr is an MultiDictView containing cookie attributes. Within
         attrs, unary attributes (e.g. HTTPOnly) are indicated by a Null value.

mitmproxy/net/http/url.py

@@ -2,7 +2,7 @@ import urllib
 from typing import Sequence
 from typing import Tuple
 
-from netlib import check
+from mitmproxy.net import check
 
 
 # PY2 workaround

mitmproxy/net/socks.py

@@ -2,8 +2,8 @@ import struct
 import array
 import ipaddress
 
-from netlib import tcp
+from mitmproxy.net import tcp
-from netlib import check
+from mitmproxy.net import check
 from mitmproxy.types import bidi
 
 

mitmproxy/net/websockets/frame.py

@@ -2,7 +2,7 @@ import os
 import struct
 import io
 
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy.utils import strutils
 from mitmproxy.utils import bits
 from mitmproxy.utils import human

mitmproxy/net/websockets/utils.py

@@ -8,7 +8,7 @@ import base64
 import hashlib
 import os
 
-from netlib import http
+from mitmproxy.net import http
 from mitmproxy.utils import strutils
 
 MAGIC = b'258EAFA5-E914-47DA-95CA-C5AB0DC85B11'

mitmproxy/net/wsgi.py

@@ -3,8 +3,8 @@ import traceback
 import urllib
 import io
 
-from netlib import http
+from mitmproxy.net import http
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy.utils import strutils
 
 

mitmproxy/proxy/config.py

@@ -11,9 +11,9 @@ from OpenSSL import SSL, crypto
 from mitmproxy import exceptions
 from mitmproxy import options as moptions
 from mitmproxy import certs
-from netlib import tcp
+from mitmproxy.net import tcp
-from netlib.http import authentication
+from mitmproxy.net.http import authentication
-from netlib.http import url
+from mitmproxy.net.http import url
 
 CONF_BASENAME = "mitmproxy"
 

mitmproxy/proxy/modes/socks_proxy.py

@@ -1,6 +1,6 @@
 from mitmproxy import exceptions
 from mitmproxy.proxy import protocol
-from netlib import socks
+from mitmproxy.net import socks
 
 
 class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin):

mitmproxy/proxy/protocol/http.py

@@ -6,9 +6,9 @@ from mitmproxy import http
 from mitmproxy import flow
 from mitmproxy.proxy.protocol import base
 from mitmproxy.proxy.protocol import websockets as pwebsockets
-import netlib.http
+import mitmproxy.net.http
-from netlib import tcp
+from mitmproxy.net import tcp
-from netlib import websockets
+from mitmproxy.net import websockets
 
 
 class _HttpTransmissionLayer(base.Layer):
@@ -422,13 +422,13 @@ class HttpLayer(base.Layer):
                     self.send_response(http.make_error_response(
                         401,
                         "Authentication Required",
-                        netlib.http.Headers(**self.config.authenticator.auth_challenge_headers())
+                        mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
                     ))
                 else:
                     self.send_response(http.make_error_response(
                         407,
                         "Proxy Authentication Required",
-                        netlib.http.Headers(**self.config.authenticator.auth_challenge_headers())
+                        mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
                     ))
                 return False
         return True

mitmproxy/proxy/protocol/http1.py

@@ -1,6 +1,6 @@
 from mitmproxy import http
 from mitmproxy.proxy.protocol import http as httpbase
-from netlib.http import http1
+from mitmproxy.net.http import http1
 
 
 class Http1Layer(httpbase._HttpTransmissionLayer):

mitmproxy/proxy/protocol/http2.py

@@ -12,10 +12,10 @@ from mitmproxy import exceptions
 from mitmproxy import http
 from mitmproxy.proxy.protocol import base
 from mitmproxy.proxy.protocol import http as httpbase
-import netlib.http
+import mitmproxy.net.http
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy.types import basethread
-from netlib.http import http2
+from mitmproxy.net.http import http2
 
 
 class SafeH2Connection(connection.H2Connection):
@@ -148,7 +148,7 @@ class Http2Layer(base.Layer):
         return True
 
     def _handle_request_received(self, eid, event, h2_connection):
-        headers = netlib.http.Headers([[k, v] for k, v in event.headers])
+        headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers])
         self.streams[eid] = Http2SingleStreamLayer(self, h2_connection, eid, headers)
         self.streams[eid].timestamp_start = time.time()
         self.streams[eid].no_body = (event.stream_ended is not None)
@@ -162,7 +162,7 @@ class Http2Layer(base.Layer):
         return True
 
     def _handle_response_received(self, eid, event):
-        headers = netlib.http.Headers([[k, v] for k, v in event.headers])
+        headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers])
         self.streams[eid].queued_data_length = 0
         self.streams[eid].timestamp_start = time.time()
         self.streams[eid].response_headers = headers
@@ -239,7 +239,7 @@ class Http2Layer(base.Layer):
             self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers)
             self.client_conn.send(self.client_conn.h2.data_to_send())
 
-        headers = netlib.http.Headers([[k, v] for k, v in event.headers])
+        headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers])
         self.streams[event.pushed_stream_id] = Http2SingleStreamLayer(self, h2_connection, event.pushed_stream_id, headers)
         self.streams[event.pushed_stream_id].timestamp_start = time.time()
         self.streams[event.pushed_stream_id].pushed = True

mitmproxy/proxy/protocol/http_replay.py

@@ -6,7 +6,7 @@ from mitmproxy import exceptions
 from mitmproxy import http
 from mitmproxy import flow
 from mitmproxy import connections
-from netlib.http import http1
+from mitmproxy.net.http import http1
 from mitmproxy.types import basethread
 
 

mitmproxy/proxy/protocol/rawtcp.py

@@ -2,7 +2,7 @@ import socket
 
 from OpenSSL import SSL
 
-import netlib.tcp
+import mitmproxy.net.tcp
 from mitmproxy import tcp
 from mitmproxy import flow
 from mitmproxy import exceptions
@@ -31,7 +31,7 @@ class RawTCPLayer(base.Layer):
 
         try:
             while not self.channel.should_exit.is_set():
-                r = netlib.tcp.ssl_read_select(conns, 10)
+                r = mitmproxy.net.tcp.ssl_read_select(conns, 10)
                 for conn in r:
                     dst = server if conn == client else client
 

mitmproxy/proxy/protocol/tls.py

@@ -6,7 +6,7 @@ import construct
 from mitmproxy import exceptions
 from mitmproxy.contrib.tls import _constructs
 from mitmproxy.proxy.protocol import base
-from netlib import check
+from mitmproxy.net import check
 
 
 # taken from https://testssl.sh/openssl-rfc.mappping.html

mitmproxy/proxy/protocol/websockets.py

@@ -4,8 +4,8 @@ from OpenSSL import SSL
 from mitmproxy import exceptions
 from mitmproxy.proxy.protocol import base
 from mitmproxy.utils import strutils
-from netlib import tcp
+from mitmproxy.net import tcp
-from netlib import websockets
+from mitmproxy.net import websockets
 
 
 class WebSocketsLayer(base.Layer):

mitmproxy/proxy/server.py

@@ -8,8 +8,8 @@ from mitmproxy import http
 from mitmproxy import log
 from mitmproxy.proxy import modes
 from mitmproxy.proxy import root_context
-from netlib import tcp
+from mitmproxy.net import tcp
-from netlib.http import http1
+from mitmproxy.net.http import http1
 
 
 class DummyServer:

mitmproxy/test/tutils.py

@@ -7,8 +7,8 @@ from contextlib import contextmanager
 import sys
 
 from mitmproxy.utils import data
-from netlib import tcp
+from mitmproxy.net import tcp
-from netlib import http
+from mitmproxy.net import http
 
 
 def treader(bytes):
@@ -89,13 +89,13 @@ class RaisesContext:
         return True
 
 
-test_data = data.Data(__name__).push("../../test/netlib")
+test_data = data.Data(__name__).push("../../test/mitmproxy/net")
 
 
 def treq(**kwargs):
     """
     Returns:
-        netlib.http.Request
+        mitmproxy.net.http.Request
     """
     default = dict(
         first_line_format="relative",
@@ -115,7 +115,7 @@ def treq(**kwargs):
 def tresp(**kwargs):
     """
     Returns:
-        netlib.http.Response
+        mitmproxy.net.http.Response
     """
     default = dict(
         http_version=b"HTTP/1.1",

mitmproxy/tools/cmdline.py

@@ -6,7 +6,7 @@ from mitmproxy import flowfilter
 from mitmproxy import options
 from mitmproxy import platform
 from mitmproxy.utils import human
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy import version
 
 

mitmproxy/tools/console/common.py

@@ -6,7 +6,7 @@ import os
 import urwid
 import urwid.util
 
-import netlib
+import mitmproxy.net
 from mitmproxy.utils import lrucache
 from mitmproxy.tools.console import signals
 from mitmproxy import export
@@ -226,7 +226,7 @@ def format_flow_data(key, scope, flow):
         if request.content is None:
             return None, "Request content is missing"
         if key == "h":
-            data += netlib.http.http1.assemble_request(request)
+            data += mitmproxy.net.http.http1.assemble_request(request)
         elif key == "c":
             data += request.get_content(strict=False)
         else:
@@ -240,7 +240,7 @@ def format_flow_data(key, scope, flow):
         if response.content is None:
             return None, "Response content is missing"
         if key == "h":
-            data += netlib.http.http1.assemble_response(response)
+            data += mitmproxy.net.http.http1.assemble_response(response)
         elif key == "c":
             data += response.get_content(strict=False)
         else:

mitmproxy/tools/console/flowlist.py

@@ -1,6 +1,6 @@
 import urwid
 
-import netlib.http.url
+import mitmproxy.net.http.url
 from mitmproxy import exceptions
 from mitmproxy.tools.console import common
 from mitmproxy.tools.console import signals
@@ -325,7 +325,7 @@ class FlowListBox(urwid.ListBox):
         )
 
     def new_request(self, url, method):
-        parts = netlib.http.url.parse(str(url))
+        parts = mitmproxy.net.http.url.parse(str(url))
         if not parts:
             signals.status_message.send(message="Invalid Url")
             return

mitmproxy/tools/console/flowview.py

@@ -16,8 +16,8 @@ from mitmproxy.tools.console import searchable
 from mitmproxy.tools.console import signals
 from mitmproxy.tools.console import tabs
 from mitmproxy import export
-from netlib.http import Headers
+from mitmproxy.net.http import Headers
-from netlib.http import status_codes
+from mitmproxy.net.http import status_codes
 
 
 class SearchError(Exception):

mitmproxy/tools/console/grideditor/col_subgrid.py

@@ -1,7 +1,7 @@
 import urwid
 from mitmproxy.tools.console.grideditor import base
 from mitmproxy.tools.console import signals
-from netlib.http import cookies
+from mitmproxy.net.http import cookies
 
 
 class Column(base.Column):

mitmproxy/tools/console/grideditor/editors.py

@@ -9,7 +9,7 @@ from mitmproxy.tools.console.grideditor import col_bytes
 from mitmproxy.tools.console.grideditor import col_text
 from mitmproxy.tools.console.grideditor import col_subgrid
 from mitmproxy.tools.console import signals
-from netlib.http import user_agents
+from mitmproxy.net.http import user_agents
 
 
 class QueryEditor(base.GridEditor):

mitmproxy/tools/console/master.py

@@ -37,7 +37,7 @@ from mitmproxy.tools.console import window
 from mitmproxy.flowfilter import FMarked
 from mitmproxy.utils import strutils
 
-from netlib import tcp
+from mitmproxy.net import tcp
 
 EVENTLOG_SIZE = 500
 

mitmproxy/tools/console/statusbar.py

@@ -2,7 +2,7 @@ import os.path
 
 import urwid
 
-import netlib.http.url
+import mitmproxy.net.http.url
 from mitmproxy.tools.console import common
 from mitmproxy.tools.console import pathedit
 from mitmproxy.tools.console import signals
@@ -208,7 +208,7 @@ class StatusBar(urwid.WidgetWrap):
 
         if self.master.options.mode in ["reverse", "upstream"]:
             dst = self.master.server.config.upstream_server
-            r.append("[dest:%s]" % netlib.http.url.unparse(
+            r.append("[dest:%s]" % mitmproxy.net.http.url.unparse(
                 dst.scheme,
                 dst.address.host,
                 dst.address.port

mitmproxy/tools/dump.py

@@ -8,7 +8,7 @@ from mitmproxy import io
 from mitmproxy import options
 from mitmproxy import master
 from mitmproxy.addons import dumper, termlog
-from netlib import tcp
+from mitmproxy.net import tcp
 
 
 class DumpError(Exception):

mitmproxy/tools/web/master.py

@@ -13,7 +13,7 @@ from mitmproxy.addons import state
 from mitmproxy import options
 from mitmproxy import master
 from mitmproxy.tools.web import app
-from netlib.http import authentication
+from mitmproxy.net.http import authentication
 
 
 class Stop(Exception):

mitmproxy/utils/debug.py

@@ -84,7 +84,7 @@ def dump_info(signal=None, frame=None, file=sys.stdout, testing=False):  # pragm
     d = {}
     for i in gc.get_objects():
         t = str(type(i))
-        if "mitmproxy" in t or "netlib" in t:
+        if "mitmproxy" in t:
             d[t] = d.setdefault(t, 0) + 1
     itms = list(d.items())
     itms.sort(key=lambda x: x[1])

mitmproxy/utils/version_check.py

@@ -1,7 +1,6 @@
 """
-Having installed a wrong version of pyOpenSSL or netlib is unfortunately a
+Having installed a wrong version of pyOpenSSL is unfortunately a very common
-very common source of error. Check before every start that both versions
+source of error. Check before every start that both versions are somewhat okay.
-are somewhat okay.
 """
 import sys
 import inspect

netlib/http/__init__.py

@@ -1,15 +0,0 @@
-from netlib.http.request import Request
-from netlib.http.response import Response
-from netlib.http.message import Message
-from netlib.http.headers import Headers, parse_content_type
-from netlib.http.message import decoded
-from netlib.http import http1, http2, status_codes, multipart
-
-__all__ = [
-    "Request",
-    "Response",
-    "Message",
-    "Headers", "parse_content_type",
-    "decoded",
-    "http1", "http2", "status_codes", "multipart",
-]

netlib/http/http2/__init__.py

@@ -1,8 +0,0 @@
-from netlib.http.http2.framereader import read_raw_frame, parse_frame
-from netlib.http.http2.utils import parse_headers
-
-__all__ = [
-    "read_raw_frame",
-    "parse_frame",
-    "parse_headers",
-]

pathod/language/http.py

@@ -2,12 +2,12 @@ import abc
 
 import pyparsing as pp
 
-from netlib.http import url
+from mitmproxy.net.http import url
-import netlib.websockets
+import mitmproxy.net.websockets
-from netlib.http import status_codes, user_agents
+from mitmproxy.net.http import status_codes, user_agents
 from . import base, exceptions, actions, message
 
-# TODO: use netlib.semantics.protocol assemble method,
+# TODO: use mitmproxy.net.semantics.protocol assemble method,
 # instead of duplicating the HTTP on-the-wire representation here.
 # see http2 language for an example
 
@@ -198,7 +198,7 @@ class Response(_HTTPMessage):
                     1,
                     StatusCode(101)
                 )
-            headers = netlib.websockets.server_handshake_headers(
+            headers = mitmproxy.net.websockets.server_handshake_headers(
                 settings.websocket_key
             )
             for i in headers.fields:
@@ -310,7 +310,7 @@ class Request(_HTTPMessage):
                     1,
                     Method("get")
                 )
-            for i in netlib.websockets.client_handshake_headers().fields:
+            for i in mitmproxy.net.websockets.client_handshake_headers().fields:
                 if not get_header(i[0], self.headers):
                     tokens.append(
                         Header(

pathod/language/http2.py

@@ -1,7 +1,7 @@
 import pyparsing as pp
 
-from netlib import http
+from mitmproxy.net import http
-from netlib.http import user_agents, Headers
+from mitmproxy.net.http import user_agents, Headers
 from . import base, message
 
 """

pathod/language/websockets.py

@@ -1,6 +1,6 @@
 import random
 import string
-import netlib.websockets
+import mitmproxy.net.websockets
 from mitmproxy.utils import strutils
 import pyparsing as pp
 from . import base, generators, actions, message
@@ -14,12 +14,12 @@ class WF(base.CaselessLiteral):
 
 class OpCode(base.IntField):
     names = {
-        "continue": netlib.websockets.OPCODE.CONTINUE,
+        "continue": mitmproxy.net.websockets.OPCODE.CONTINUE,
-        "text": netlib.websockets.OPCODE.TEXT,
+        "text": mitmproxy.net.websockets.OPCODE.TEXT,
-        "binary": netlib.websockets.OPCODE.BINARY,
+        "binary": mitmproxy.net.websockets.OPCODE.BINARY,
-        "close": netlib.websockets.OPCODE.CLOSE,
+        "close": mitmproxy.net.websockets.OPCODE.CLOSE,
-        "ping": netlib.websockets.OPCODE.PING,
+        "ping": mitmproxy.net.websockets.OPCODE.PING,
-        "pong": netlib.websockets.OPCODE.PONG,
+        "pong": mitmproxy.net.websockets.OPCODE.PONG,
     }
     max = 15
     preamble = "c"
@@ -215,11 +215,11 @@ class WebsocketFrame(message.Message):
             v = getattr(self, i, None)
             if v is not None:
                 frameparts[i] = v.value
-        frame = netlib.websockets.FrameHeader(**frameparts)
+        frame = mitmproxy.net.websockets.FrameHeader(**frameparts)
         vals = [bytes(frame)]
         if bodygen:
             if frame.masking_key and not self.rawbody:
-                masker = netlib.websockets.Masker(frame.masking_key)
+                masker = mitmproxy.net.websockets.Masker(frame.masking_key)
                 vals.append(
                     generators.TransformGenerator(
                         bodygen,

pathod/pathoc.py

@@ -13,12 +13,12 @@ import logging
 
 from mitmproxy.test.tutils import treq
 from mitmproxy.utils import strutils
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy import certs
-from netlib import websockets
+from mitmproxy.net import websockets
-from netlib import socks
+from mitmproxy.net import socks
 from mitmproxy import exceptions
-from netlib.http import http1
+from mitmproxy.net.http import http1
 from mitmproxy.types import basethread
 
 from pathod import log

pathod/pathoc_cmdline.py

@@ -3,9 +3,9 @@ import argparse
 import os
 import os.path
 
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy import version
-from netlib.http import user_agents
+from mitmproxy.net.http import user_agents
 from . import pathoc, language
 
 

pathod/pathod.py

@@ -4,9 +4,9 @@ import os
 import sys
 import threading
 
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy import certs as mcerts
-from netlib import websockets
+from mitmproxy.net import websockets
 from mitmproxy import version
 
 import urllib

pathod/pathod_cmdline.py

@@ -4,7 +4,7 @@ import os
 import os.path
 import re
 
-from netlib import tcp
+from mitmproxy.net import tcp
 from mitmproxy.utils import human
 from mitmproxy import version
 from . import pathod

pathod/protocols/http.py

@@ -1,6 +1,6 @@
 from mitmproxy import version
 from mitmproxy import exceptions
-from netlib.http import http1
+from mitmproxy.net.http import http1
 from .. import language
 
 

pathod/protocols/http2.py

@@ -4,10 +4,10 @@ import time
 import hyperframe.frame
 from hpack.hpack import Encoder, Decoder
 
-from netlib.http import http2
+from mitmproxy.net.http import http2
-import netlib.http.headers
+import mitmproxy.net.http.headers
-import netlib.http.response
+import mitmproxy.net.http.response
-import netlib.http.request
+import mitmproxy.net.http.request
 from mitmproxy.types import bidi
 
 from .. import language
@@ -100,7 +100,7 @@ class HTTP2StateProtocol:
 
         first_line_format, method, scheme, host, port, path = http2.parse_headers(headers)
 
-        request = netlib.http.request.Request(
+        request = mitmproxy.net.http.request.Request(
             first_line_format,
             method,
             scheme,
@@ -148,7 +148,7 @@ class HTTP2StateProtocol:
         else:
             timestamp_end = None
 
-        response = netlib.http.response.Response(
+        response = mitmproxy.net.http.response.Response(
             b"HTTP/2.0",
             int(headers.get(':status', 502)),
             b'',
@@ -162,15 +162,15 @@ class HTTP2StateProtocol:
         return response
 
     def assemble(self, message):
-        if isinstance(message, netlib.http.request.Request):
+        if isinstance(message, mitmproxy.net.http.request.Request):
             return self.assemble_request(message)
-        elif isinstance(message, netlib.http.response.Response):
+        elif isinstance(message, mitmproxy.net.http.response.Response):
             return self.assemble_response(message)
         else:
             raise ValueError("HTTP message not supported.")
 
     def assemble_request(self, request):
-        assert isinstance(request, netlib.http.request.Request)
+        assert isinstance(request, mitmproxy.net.http.request.Request)
 
         authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host
         if self.tcp_handler.address.port != 443:
@@ -194,7 +194,7 @@ class HTTP2StateProtocol:
             self._create_body(request.body, stream_id)))
 
     def assemble_response(self, response):
-        assert isinstance(response, netlib.http.response.Response)
+        assert isinstance(response, mitmproxy.net.http.response.Response)
 
         headers = response.headers.copy()
 
@@ -394,7 +394,7 @@ class HTTP2StateProtocol:
             else:
                 self._handle_unexpected_frame(frm)
 
-        headers = netlib.http.headers.Headers(
+        headers = mitmproxy.net.http.headers.Headers(
             [[k, v] for k, v in self.decoder.decode(header_blocks, raw=True)]
         )
 

pathod/protocols/websockets.py

@@ -1,6 +1,6 @@
 import time
 
-from netlib import websockets
+from mitmproxy.net import websockets
 from pathod import language
 from mitmproxy import exceptions
 

release/README.mkd

@@ -7,7 +7,7 @@
   - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release
     for a single project. This matches with what we do in `setup.py`:
 
-        "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)
+        "mitmproxy>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)
 
 
 # Release Checklist

release/rtool.py

@@ -48,7 +48,7 @@ VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
 VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
 
 # Project Configuration
-VERSION_FILE = join(ROOT_DIR, "netlib", "version.py")
+VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
 PROJECT_NAME = "mitmproxy"
 PYTHON_VERSION = "py2.py3"
 BDISTS = {

setup.py

@@ -44,7 +44,6 @@ setup(
     packages=find_packages(include=[
         "mitmproxy", "mitmproxy.*",
         "pathod", "pathod.*",
-        "netlib", "netlib.*"
     ]),
     include_package_data=True,
     entry_points={