mitmproxy/netlib/wsgi.py

from __future__ import (absolute_import, print_function, division)
from io import BytesIO, StringIO
import urllib
import time
import traceback

import six
from six.moves import urllib

from netlib.utils import always_bytes, native
from . import http, tcp

class ClientConn(object):

    def __init__(self, address):
        self.address = tcp.Address.wrap(address)


class Flow(object):

    def __init__(self, address, request):
        self.client_conn = ClientConn(address)
        self.request = request


class Request(object):

    def __init__(self, scheme, method, path, http_version, headers, content):
        self.scheme, self.method, self.path = scheme, method, path
        self.headers, self.content = headers, content
        self.http_version = http_version


def date_time_string():
    """Return the current date and time formatted for a message header."""
    WEEKS = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
    MONTHS = [
        None,
        'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
        'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'
    ]
    now = time.time()
    year, month, day, hh, mm, ss, wd, y_, z_ = time.gmtime(now)
    s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
        WEEKS[wd],
        day, MONTHS[month], year,
        hh, mm, ss
    )
    return s


class WSGIAdaptor(object):

    def __init__(self, app, domain, port, sversion):
        self.app, self.domain, self.port, self.sversion = app, domain, port, sversion

    def make_environ(self, flow, errsoc, **extra):
        path = native(flow.request.path, "latin-1")
        if '?' in path:
            path_info, query = native(path, "latin-1").split('?', 1)
        else:
            path_info = path
            query = ''
        environ = {
            'wsgi.version': (1, 0),
            'wsgi.url_scheme': native(flow.request.scheme, "latin-1"),
            'wsgi.input': BytesIO(flow.request.content or b""),
            'wsgi.errors': errsoc,
            'wsgi.multithread': True,
            'wsgi.multiprocess': False,
            'wsgi.run_once': False,
            'SERVER_SOFTWARE': self.sversion,
            'REQUEST_METHOD': native(flow.request.method, "latin-1"),
            'SCRIPT_NAME': '',
            'PATH_INFO': urllib.parse.unquote(path_info),
            'QUERY_STRING': query,
            'CONTENT_TYPE': native(flow.request.headers.get('Content-Type', ''), "latin-1"),
            'CONTENT_LENGTH': native(flow.request.headers.get('Content-Length', ''), "latin-1"),
            'SERVER_NAME': self.domain,
            'SERVER_PORT': str(self.port),
            'SERVER_PROTOCOL': native(flow.request.http_version, "latin-1"),
        }
        environ.update(extra)
        if flow.client_conn.address:
            environ["REMOTE_ADDR"] = native(flow.client_conn.address.host, "latin-1")
            environ["REMOTE_PORT"] = flow.client_conn.address.port

        for key, value in flow.request.headers.items():
            key = 'HTTP_' + native(key, "latin-1").upper().replace('-', '_')
            if key not in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'):
                environ[key] = value
        return environ

    def error_page(self, soc, headers_sent, s):
        """
            Make a best-effort attempt to write an error page. If headers are
            already sent, we just bung the error into the page.
        """
        c = """
            <html>
                <h1>Internal Server Error</h1>
                <pre>{err}"</pre>
            </html>
        """.format(err=s).strip().encode()

        if not headers_sent:
            soc.write(b"HTTP/1.1 500 Internal Server Error\r\n")
            soc.write(b"Content-Type: text/html\r\n")
            soc.write("Content-Length: {length}\r\n".format(length=len(c)).encode())
            soc.write(b"\r\n")
        soc.write(c)

    def serve(self, request, soc, **env):
        state = dict(
            response_started=False,
            headers_sent=False,
            status=None,
            headers=None
        )

        def write(data):
            if not state["headers_sent"]:
                soc.write("HTTP/1.1 {status}\r\n".format(status=state["status"]).encode())
                headers = state["headers"]
                if 'server' not in headers:
                    headers["Server"] = self.sversion
                if 'date' not in headers:
                    headers["Date"] = date_time_string()
                soc.write(bytes(headers))
                soc.write(b"\r\n")
                state["headers_sent"] = True
            if data:
                soc.write(data)
            soc.flush()

        def start_response(status, headers, exc_info=None):
            if exc_info:
                if state["headers_sent"]:
                    six.reraise(*exc_info)
            elif state["status"]:
                raise AssertionError('Response already started')
            state["status"] = status
            state["headers"] = http.Headers([[always_bytes(k), always_bytes(v)] for k,v in headers])
            if exc_info:
                self.error_page(soc, state["headers_sent"], traceback.format_tb(exc_info[2]))
                state["headers_sent"] = True

        errs = six.BytesIO()
        try:
            dataiter = self.app(
                self.make_environ(request, errs, **env), start_response
            )
            for i in dataiter:
                write(i)
            if not state["headers_sent"]:
                write(b"")
        except Exception as e:
            try:
                s = traceback.format_exc()
                errs.write(s.encode("utf-8", "replace"))
                self.error_page(soc, state["headers_sent"], s)
            except Exception:    # pragma: no cover
                pass
        return errs.getvalue()
minor cleanups 2014-08-16 13:53:07 +00:00			`from __future__ import (absolute_import, print_function, division)`
python 3++ 2015-09-20 22:44:17 +00:00			`from io import BytesIO, StringIO`
ODict improvements - Setting values now tries to preserve the existing order, rather than just appending to the end. - __repr__ now returns a repr of the tuple list. The old repr becomes a .format() method. This is clearer, makes troubleshooting easier, and doesn't assume all data in ODicts are header-like 2015-04-14 22:28:17 +00:00			`import urllib`
			`import time`
			`import traceback`
python3++ 2015-09-20 16:12:55 +00:00
			`import six`
python 3++ 2015-09-20 22:44:17 +00:00			`from six.moves import urllib`
python3++ 2015-09-20 16:12:55 +00:00
python 3++ 2015-09-20 22:44:17 +00:00			`from netlib.utils import always_bytes, native`
headers: adjust everything 2015-09-05 16:15:47 +00:00			`from . import http, tcp`
Import wsgi. 2012-06-18 22:42:25 +00:00
fix code smell 2015-04-09 00:09:33 +00:00			`class ClientConn(object):`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00
Adapt WSGI, convert test suite to nose. 2012-06-19 02:23:22 +00:00			`def __init__(self, address):`
adjust netlib.wsgi to reflect changes in mitmproxys flow format 2014-02-05 20:34:14 +00:00			`self.address = tcp.Address.wrap(address)`


fix code smell 2015-04-09 00:09:33 +00:00			`class Flow(object):`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00
adapt netlib.wsgi to changes in mitmproxy/mitmproxy#341 2014-09-03 15:15:50 +00:00			`def __init__(self, address, request):`
			`self.client_conn = ClientConn(address)`
			`self.request = request`
Adapt WSGI, convert test suite to nose. 2012-06-19 02:23:22 +00:00

fix code smell 2015-04-09 00:09:33 +00:00			`class Request(object):`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00
refactor response model 2015-09-26 15:39:50 +00:00			`def __init__(self, scheme, method, path, http_version, headers, content):`
Adapt WSGI, convert test suite to nose. 2012-06-19 02:23:22 +00:00			`self.scheme, self.method, self.path = scheme, method, path`
refactor response model 2015-09-26 15:39:50 +00:00			`self.headers, self.content = headers, content`
python 3++ 2015-09-20 22:44:17 +00:00			`self.http_version = http_version`
Adapt WSGI, convert test suite to nose. 2012-06-19 02:23:22 +00:00

Import wsgi. 2012-06-18 22:42:25 +00:00			`def date_time_string():`
			`"""Return the current date and time formatted for a message header."""`
			`WEEKS = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']`
ODict improvements - Setting values now tries to preserve the existing order, rather than just appending to the end. - __repr__ now returns a repr of the tuple list. The old repr becomes a .format() method. This is clearer, makes troubleshooting easier, and doesn't assume all data in ODicts are header-like 2015-04-14 22:28:17 +00:00			`MONTHS = [`
			`None,`
			`'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',`
			`'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'`
			`]`
Import wsgi. 2012-06-18 22:42:25 +00:00			`now = time.time()`
mark unused variables and arguments 2015-06-18 13:32:52 +00:00			`year, month, day, hh, mm, ss, wd, y_, z_ = time.gmtime(now)`
Import wsgi. 2012-06-18 22:42:25 +00:00			`s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (`
ODict improvements - Setting values now tries to preserve the existing order, rather than just appending to the end. - __repr__ now returns a repr of the tuple list. The old repr becomes a .format() method. This is clearer, makes troubleshooting easier, and doesn't assume all data in ODicts are header-like 2015-04-14 22:28:17 +00:00			`WEEKS[wd],`
			`day, MONTHS[month], year,`
			`hh, mm, ss`
			`)`
Import wsgi. 2012-06-18 22:42:25 +00:00			`return s`


fix code smell 2015-04-09 00:09:33 +00:00			`class WSGIAdaptor(object):`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00
Import wsgi. 2012-06-18 22:42:25 +00:00			`def __init__(self, app, domain, port, sversion):`
			`self.app, self.domain, self.port, self.sversion = app, domain, port, sversion`

adapt netlib.wsgi to changes in mitmproxy/mitmproxy#341 2014-09-03 15:15:50 +00:00			`def make_environ(self, flow, errsoc, **extra):`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`path = native(flow.request.path, "latin-1")`
python 3++ 2015-09-20 22:44:17 +00:00			`if '?' in path:`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`path_info, query = native(path, "latin-1").split('?', 1)`
Import wsgi. 2012-06-18 22:42:25 +00:00			`else:`
python 3++ 2015-09-20 22:44:17 +00:00			`path_info = path`
Import wsgi. 2012-06-18 22:42:25 +00:00			`query = ''`
			`environ = {`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00			`'wsgi.version': (1, 0),`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`'wsgi.url_scheme': native(flow.request.scheme, "latin-1"),`
refactor response model 2015-09-26 15:39:50 +00:00			`'wsgi.input': BytesIO(flow.request.content or b""),`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00			`'wsgi.errors': errsoc,`
			`'wsgi.multithread': True,`
			`'wsgi.multiprocess': False,`
			`'wsgi.run_once': False,`
			`'SERVER_SOFTWARE': self.sversion,`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`'REQUEST_METHOD': native(flow.request.method, "latin-1"),`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00			`'SCRIPT_NAME': '',`
python 3++ 2015-09-20 22:44:17 +00:00			`'PATH_INFO': urllib.parse.unquote(path_info),`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00			`'QUERY_STRING': query,`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`'CONTENT_TYPE': native(flow.request.headers.get('Content-Type', ''), "latin-1"),`
			`'CONTENT_LENGTH': native(flow.request.headers.get('Content-Length', ''), "latin-1"),`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00			`'SERVER_NAME': self.domain,`
			`'SERVER_PORT': str(self.port),`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`'SERVER_PROTOCOL': native(flow.request.http_version, "latin-1"),`
Import wsgi. 2012-06-18 22:42:25 +00:00			`}`
Make it possible to pass custom environment variables into wsgi apps. 2014-01-04 21:57:50 +00:00			`environ.update(extra)`
adapt netlib.wsgi to changes in mitmproxy/mitmproxy#341 2014-09-03 15:15:50 +00:00			`if flow.client_conn.address:`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`environ["REMOTE_ADDR"] = native(flow.client_conn.address.host, "latin-1")`
python 3++ 2015-09-20 22:44:17 +00:00			`environ["REMOTE_PORT"] = flow.client_conn.address.port`
Import wsgi. 2012-06-18 22:42:25 +00:00
adapt netlib.wsgi to changes in mitmproxy/mitmproxy#341 2014-09-03 15:15:50 +00:00			`for key, value in flow.request.headers.items():`
minor encoding fixes 2015-09-21 16:34:43 +00:00			`key = 'HTTP_' + native(key, "latin-1").upper().replace('-', '_')`
Import wsgi. 2012-06-18 22:42:25 +00:00			`if key not in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'):`
			`environ[key] = value`
			`return environ`

			`def error_page(self, soc, headers_sent, s):`
			`"""`
			`Make a best-effort attempt to write an error page. If headers are`
			`already sent, we just bung the error into the page.`
			`"""`
Porting to Python 3.4 Updated wsgi to support Python 3.4 byte strings. Updated test_wsgi to remove py.test warning for TestApp having an __init__ constructor. samc$ sudo py.test netlib/test/test_wsgi.py -r w = test session starts = platform darwin -- Python 3.4.1, pytest-2.8.2, py-1.4.30, pluggy-0.3.1 rootdir: /Users/samc/src/python/netlib, inifile: collected 6 items netlib/test/test_wsgi.py ...... = 6 passed in 0.20 seconds = 2015-11-12 02:27:10 +00:00			`c = """`
Import wsgi. 2012-06-18 22:42:25 +00:00			`<html>`
			`<h1>Internal Server Error</h1>`
Porting to Python 3.4 Updated wsgi to support Python 3.4 byte strings. Updated test_wsgi to remove py.test warning for TestApp having an __init__ constructor. samc$ sudo py.test netlib/test/test_wsgi.py -r w = test session starts = platform darwin -- Python 3.4.1, pytest-2.8.2, py-1.4.30, pluggy-0.3.1 rootdir: /Users/samc/src/python/netlib, inifile: collected 6 items netlib/test/test_wsgi.py ...... = 6 passed in 0.20 seconds = 2015-11-12 02:27:10 +00:00			`<pre>{err}"</pre>`
Import wsgi. 2012-06-18 22:42:25 +00:00			`</html>`
Porting to Python 3.4 Updated wsgi to support Python 3.4 byte strings. Updated test_wsgi to remove py.test warning for TestApp having an __init__ constructor. samc$ sudo py.test netlib/test/test_wsgi.py -r w = test session starts = platform darwin -- Python 3.4.1, pytest-2.8.2, py-1.4.30, pluggy-0.3.1 rootdir: /Users/samc/src/python/netlib, inifile: collected 6 items netlib/test/test_wsgi.py ...... = 6 passed in 0.20 seconds = 2015-11-12 02:27:10 +00:00			`""".format(err=s).strip().encode()`

Import wsgi. 2012-06-18 22:42:25 +00:00			`if not headers_sent:`
python3++ 2015-09-20 16:12:55 +00:00			`soc.write(b"HTTP/1.1 500 Internal Server Error\r\n")`
			`soc.write(b"Content-Type: text/html\r\n")`
Porting to Python 3.4 Updated wsgi to support Python 3.4 byte strings. Updated test_wsgi to remove py.test warning for TestApp having an __init__ constructor. samc$ sudo py.test netlib/test/test_wsgi.py -r w = test session starts = platform darwin -- Python 3.4.1, pytest-2.8.2, py-1.4.30, pluggy-0.3.1 rootdir: /Users/samc/src/python/netlib, inifile: collected 6 items netlib/test/test_wsgi.py ...... = 6 passed in 0.20 seconds = 2015-11-12 02:27:10 +00:00			`soc.write("Content-Length: {length}\r\n".format(length=len(c)).encode())`
python3++ 2015-09-20 16:12:55 +00:00			`soc.write(b"\r\n")`
Import wsgi. 2012-06-18 22:42:25 +00:00			`soc.write(c)`

Make it possible to pass custom environment variables into wsgi apps. 2014-01-04 21:57:50 +00:00			`def serve(self, request, soc, **env):`
Import wsgi. 2012-06-18 22:42:25 +00:00			`state = dict(`
cleanup code with autopep8 run the following command: $ autopep8 -i -r -a -a . 2015-05-27 09:18:54 +00:00			`response_started=False,`
			`headers_sent=False,`
			`status=None,`
			`headers=None`
Import wsgi. 2012-06-18 22:42:25 +00:00			`)`
ODict improvements - Setting values now tries to preserve the existing order, rather than just appending to the end. - __repr__ now returns a repr of the tuple list. The old repr becomes a .format() method. This is clearer, makes troubleshooting easier, and doesn't assume all data in ODicts are header-like 2015-04-14 22:28:17 +00:00
Import wsgi. 2012-06-18 22:42:25 +00:00			`def write(data):`
			`if not state["headers_sent"]:`
Porting to Python 3.4 Updated wsgi to support Python 3.4 byte strings. Updated test_wsgi to remove py.test warning for TestApp having an __init__ constructor. samc$ sudo py.test netlib/test/test_wsgi.py -r w = test session starts = platform darwin -- Python 3.4.1, pytest-2.8.2, py-1.4.30, pluggy-0.3.1 rootdir: /Users/samc/src/python/netlib, inifile: collected 6 items netlib/test/test_wsgi.py ...... = 6 passed in 0.20 seconds = 2015-11-12 02:27:10 +00:00			`soc.write("HTTP/1.1 {status}\r\n".format(status=state["status"]).encode())`
headers: adjust everything 2015-09-05 16:15:47 +00:00			`headers = state["headers"]`
			`if 'server' not in headers:`
			`headers["Server"] = self.sversion`
			`if 'date' not in headers:`
			`headers["Date"] = date_time_string()`
python3++ 2015-09-20 16:12:55 +00:00			`soc.write(bytes(headers))`
			`soc.write(b"\r\n")`
Import wsgi. 2012-06-18 22:42:25 +00:00			`state["headers_sent"] = True`
Hunt down a tricky WSGI socket hang. 2012-06-26 02:49:23 +00:00			`if data:`
			`soc.write(data)`
Import wsgi. 2012-06-18 22:42:25 +00:00			`soc.flush()`

			`def start_response(status, headers, exc_info=None):`
			`if exc_info:`
python 3++ 2015-09-20 22:44:17 +00:00			`if state["headers_sent"]:`
			`six.reraise(*exc_info)`
Import wsgi. 2012-06-18 22:42:25 +00:00			`elif state["status"]:`
			`raise AssertionError('Response already started')`
			`state["status"] = status`
python 3++ 2015-09-20 22:44:17 +00:00			`state["headers"] = http.Headers([[always_bytes(k), always_bytes(v)] for k,v in headers])`
			`if exc_info:`
			`self.error_page(soc, state["headers_sent"], traceback.format_tb(exc_info[2]))`
			`state["headers_sent"] = True`
Import wsgi. 2012-06-18 22:42:25 +00:00
python 3++ 2015-09-20 22:44:17 +00:00			`errs = six.BytesIO()`
Import wsgi. 2012-06-18 22:42:25 +00:00			`try:`
ODict improvements - Setting values now tries to preserve the existing order, rather than just appending to the end. - __repr__ now returns a repr of the tuple list. The old repr becomes a .format() method. This is clearer, makes troubleshooting easier, and doesn't assume all data in ODicts are header-like 2015-04-14 22:28:17 +00:00			`dataiter = self.app(`
			`self.make_environ(request, errs, **env), start_response`
			`)`
Import wsgi. 2012-06-18 22:42:25 +00:00			`for i in dataiter:`
			`write(i)`
			`if not state["headers_sent"]:`
python3++ 2015-09-20 16:12:55 +00:00			`write(b"")`
headers: adjust everything 2015-09-05 16:15:47 +00:00			`except Exception as e:`
Import wsgi. 2012-06-18 22:42:25 +00:00			`try:`
			`s = traceback.format_exc()`
python 3++ 2015-09-20 22:44:17 +00:00			`errs.write(s.encode("utf-8", "replace"))`
Import wsgi. 2012-06-18 22:42:25 +00:00			`self.error_page(soc, state["headers_sent"], s)`
More accurate description of an HTTP read error, make pyflakes happy. 2013-02-23 22:08:43 +00:00			`except Exception: # pragma: no cover`
			`pass`
Import wsgi. 2012-06-18 22:42:25 +00:00			`return errs.getvalue()`