mitmproxy/pathod/libpathod/pathod.py

import copy
import logging
import os
import sys
import threading
import urllib

from netlib import tcp, http, certutils, websockets
from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \
    TlsException

from . import version, app, language, utils, log, protocols
import language.http
import language.actions
import language.exceptions
import language.websockets


DEFAULT_CERT_DOMAIN = "pathod.net"
CONFDIR = "~/.mitmproxy"
CERTSTORE_BASENAME = "mitmproxy"
CA_CERT_NAME = "mitmproxy-ca.pem"
DEFAULT_CRAFT_ANCHOR = "/p/"

logger = logging.getLogger('pathod')


class PathodError(Exception):
    pass


class SSLOptions(object):
    def __init__(
        self,
        confdir=CONFDIR,
        cn=None,
        sans=(),
        not_after_connect=None,
        request_client_cert=False,
        ssl_version=tcp.SSL_DEFAULT_METHOD,
        ssl_options=tcp.SSL_DEFAULT_OPTIONS,
        ciphers=None,
        certs=None,
        alpn_select=b'h2',
    ):
        self.confdir = confdir
        self.cn = cn
        self.sans = sans
        self.not_after_connect = not_after_connect
        self.request_client_cert = request_client_cert
        self.ssl_version = ssl_version
        self.ssl_options = ssl_options
        self.ciphers = ciphers
        self.alpn_select = alpn_select
        self.certstore = certutils.CertStore.from_store(
            os.path.expanduser(confdir),
            CERTSTORE_BASENAME
        )
        for i in certs or []:
            self.certstore.add_cert_file(*i)

    def get_cert(self, name):
        if self.cn:
            name = self.cn
        elif not name:
            name = DEFAULT_CERT_DOMAIN
        return self.certstore.get_cert(name, self.sans)


class PathodHandler(tcp.BaseHandler):
    wbufsize = 0
    sni = None

    def __init__(
        self,
        connection,
        address,
        server,
        logfp,
        settings,
        http2_framedump=False
    ):
        tcp.BaseHandler.__init__(self, connection, address, server)
        self.logfp = logfp
        self.settings = copy.copy(settings)
        self.protocol = None
        self.use_http2 = False
        self.http2_framedump = http2_framedump

    def handle_sni(self, connection):
        self.sni = connection.get_servername()

    def http_serve_crafted(self, crafted, logctx):
        error, crafted = self.server.check_policy(
            crafted, self.settings
        )
        if error:
            err = self.make_http_error_response(error)
            language.serve(err, self.wfile, self.settings)
            return None, dict(
                type="error",
                msg=error
            )

        if self.server.explain and not hasattr(crafted, 'is_error_response'):
            crafted = crafted.freeze(self.settings)
            logctx(">> Spec: %s" % crafted.spec())

        response_log = language.serve(
            crafted,
            self.wfile,
            self.settings
        )
        if response_log["disconnect"]:
            return None, response_log
        return self.handle_http_request, response_log


    def handle_http_request(self, logger):
        """
            Returns a (handler, log) tuple.

            handler: Handler for the next request, or None to disconnect
            log: A dictionary, or None
        """
        with logger.ctx() as lg:
            try:
                req = self.protocol.read_request(self.rfile)
            except HttpReadDisconnect:
                return None, None
            except HttpException as s:
                s = str(s)
                lg(s)
                return None, dict(type="error", msg=s)

            if req.method == 'CONNECT':
                return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg)

            method = req.method
            path = req.path
            http_version = req.http_version
            headers = req.headers
            body = req.content

            clientcert = None
            if self.clientcert:
                clientcert = dict(
                    cn=self.clientcert.cn,
                    subject=self.clientcert.subject,
                    serial=self.clientcert.serial,
                    notbefore=self.clientcert.notbefore.isoformat(),
                    notafter=self.clientcert.notafter.isoformat(),
                    keyinfo=self.clientcert.keyinfo,
                )

            retlog = dict(
                type="crafted",
                protocol="http",
                request=dict(
                    path=path,
                    method=method,
                    headers=headers.fields,
                    http_version=http_version,
                    sni=self.sni,
                    remote_address=self.address(),
                    clientcert=clientcert,
                ),
                cipher=None,
            )
            if self.ssl_established:
                retlog["cipher"] = self.get_current_cipher()

            m = utils.MemBool()
            websocket_key = websockets.WebsocketsProtocol.check_client_handshake(headers)
            self.settings.websocket_key = websocket_key

            # If this is a websocket initiation, we respond with a proper
            # server response, unless over-ridden.
            if websocket_key:
                anchor_gen = language.parse_pathod("ws")
            else:
                anchor_gen = None

            for regex, spec in self.server.anchors:
                if regex.match(path):
                    anchor_gen = language.parse_pathod(spec, self.use_http2)
                    break
            else:
                if m(path.startswith(self.server.craftanchor)):
                    spec = urllib.unquote(path)[len(self.server.craftanchor):]
                    if spec:
                        try:
                            anchor_gen = language.parse_pathod(spec, self.use_http2)
                        except language.ParseException as v:
                            lg("Parse error: %s" % v.msg)
                            anchor_gen = iter([self.make_http_error_response(
                                "Parse Error",
                                "Error parsing response spec: %s\n" % (
                                    v.msg + v.marked()
                                )
                            )])
                else:
                    if self.use_http2:
                        anchor_gen = iter([self.make_http_error_response(
                            "Spec Error",
                            "HTTP/2 only supports request/response with the craft anchor point: %s" %
                            self.server.craftanchor
                        )])

            if anchor_gen:
                spec = anchor_gen.next()

                if self.use_http2 and isinstance(spec, language.http2.Response):
                    spec.stream_id = req.stream_id

                lg("crafting spec: %s" % spec)
                nexthandler, retlog["response"] = self.http_serve_crafted(
                    spec,
                    lg
                )
                if nexthandler and websocket_key:
                    self.protocol = protocols.websockets.WebsocketsProtocol(self)
                    return self.protocol.handle_websocket, retlog
                else:
                    return nexthandler, retlog
            else:
                return self.protocol.handle_http_app(method, path, headers, body, lg)

    def make_http_error_response(self, reason, body=None):
        resp = self.protocol.make_error_response(reason, body)
        resp.is_error_response = True
        return resp

    def handle(self):
        self.settimeout(self.server.timeout)

        if self.server.ssl:
            try:
                cert, key, _ = self.server.ssloptions.get_cert(None)
                self.convert_to_ssl(
                    cert,
                    key,
                    handle_sni=self.handle_sni,
                    request_client_cert=self.server.ssloptions.request_client_cert,
                    cipher_list=self.server.ssloptions.ciphers,
                    method=self.server.ssloptions.ssl_version,
                    options=self.server.ssloptions.ssl_options,
                    alpn_select=self.server.ssloptions.alpn_select,
                )
            except TlsException as v:
                s = str(v)
                self.server.add_log(
                    dict(
                        type="error",
                        msg=s
                    )
                )
                log.write_raw(self.logfp, s)
                return

            alp = self.get_alpn_proto_negotiated()
            if alp == b'h2':
                self.protocol = protocols.http2.HTTP2Protocol(self)
                self.use_http2 = True

        if not self.protocol:
            self.protocol = protocols.http.HTTPProtocol(self)

        lr = self.rfile if self.server.logreq else None
        lw = self.wfile if self.server.logresp else None
        logger = log.ConnectionLogger(self.logfp, self.server.hexdump, lr, lw)

        self.settings.protocol = self.protocol

        handler = self.handle_http_request

        while not self.finished:
            handler, l = handler(logger)
            if l:
                self.addlog(l)
            if not handler:
                return

    def addlog(self, log):
        # FIXME: The bytes in the log should not be escaped. We do this at the
        # moment because JSON encoding can't handle binary data, and I don't
        # want to base64 everything.
        if self.server.logreq:
            encoded_bytes = self.rfile.get_log().encode("string_escape")
            log["request_bytes"] = encoded_bytes
        if self.server.logresp:
            encoded_bytes = self.wfile.get_log().encode("string_escape")
            log["response_bytes"] = encoded_bytes
        self.server.add_log(log)


class Pathod(tcp.TCPServer):
    LOGBUF = 500

    def __init__(
        self,
        addr,
        ssl=False,
        ssloptions=None,
        craftanchor=DEFAULT_CRAFT_ANCHOR,
        staticdir=None,
        anchors=(),
        sizelimit=None,
        noweb=False,
        nocraft=False,
        noapi=False,
        nohang=False,
        timeout=None,
        logreq=False,
        logresp=False,
        explain=False,
        hexdump=False,
        http2_framedump=False,
        webdebug=False,
        logfp=sys.stdout,
    ):
        """
            addr: (address, port) tuple. If port is 0, a free port will be
            automatically chosen.
            ssloptions: an SSLOptions object.
            craftanchor: URL prefix specifying the path under which to anchor
            response generation.
            staticdir: path to a directory of static resources, or None.
            anchors: List of (regex object, language.Request object) tuples, or
            None.
            sizelimit: Limit size of served data.
            nocraft: Disable response crafting.
            noapi: Disable the API.
            nohang: Disable pauses.
        """
        tcp.TCPServer.__init__(self, addr)
        self.ssl = ssl
        self.ssloptions = ssloptions or SSLOptions()
        self.staticdir = staticdir
        self.craftanchor = craftanchor
        self.sizelimit = sizelimit
        self.noweb, self.nocraft = noweb, nocraft
        self.noapi, self.nohang = noapi, nohang
        self.timeout, self.logreq = timeout, logreq
        self.logresp, self.hexdump = logresp, hexdump
        self.http2_framedump = http2_framedump
        self.explain = explain
        self.logfp = logfp

        self.app = app.make_app(noapi, webdebug)
        self.app.config["pathod"] = self
        self.log = []
        self.logid = 0
        self.anchors = anchors

        self.settings = language.Settings(
            staticdir=self.staticdir
        )

    def check_policy(self, req, settings):
        """
            A policy check that verifies the request size is within limits.
        """
        if self.nocraft:
            return "Crafting disabled.", None
        try:
            req = req.resolve(settings)
            l = req.maximum_length(settings)
        except language.FileAccessDenied:
            return "File access denied.", None
        if self.sizelimit and l > self.sizelimit:
            return "Response too large.", None
        pauses = [isinstance(i, language.actions.PauseAt) for i in req.actions]
        if self.nohang and any(pauses):
            return "Pauses have been disabled.", None
        return None, req

    def handle_client_connection(self, request, client_address):
        h = PathodHandler(
            request,
            client_address,
            self,
            self.logfp,
            self.settings,
            self.http2_framedump,
        )
        try:
            h.handle()
            h.finish()
        except TcpDisconnect:  # pragma: no cover
            log.write_raw(self.logfp, "Disconnect")
            self.add_log(
                dict(
                    type="error",
                    msg="Disconnect"
                )
            )
            return
        except TcpTimeout:
            log.write_raw(self.logfp, "Timeout")
            self.add_log(
                dict(
                    type="timeout",
                )
            )
            return

    def add_log(self, d):
        if not self.noapi:
            lock = threading.Lock()
            with lock:
                d["id"] = self.logid
                self.log.insert(0, d)
                if len(self.log) > self.LOGBUF:
                    self.log.pop()
                self.logid += 1
            return d["id"]

    def clear_log(self):
        lock = threading.Lock()
        with lock:
            self.log = []

    def log_by_id(self, identifier):
        for i in self.log:
            if i["id"] == identifier:
                return i

    def get_log(self):
        return self.log


def main(args):  # pragma: nocover
    ssloptions = SSLOptions(
        cn=args.cn,
        confdir=args.confdir,
        not_after_connect=args.ssl_not_after_connect,
        ciphers=args.ciphers,
        ssl_version=args.ssl_version,
        ssl_options=args.ssl_options,
        certs=args.ssl_certs,
        sans=args.sans,
    )

    root = logging.getLogger()
    if root.handlers:
        for handler in root.handlers:
            root.removeHandler(handler)

    log = logging.getLogger('pathod')
    log.setLevel(logging.DEBUG)
    fmt = logging.Formatter(
        '%(asctime)s: %(message)s',
        datefmt='%d-%m-%y %H:%M:%S',
    )
    if args.logfile:
        fh = logging.handlers.WatchedFileHandler(args.logfile)
        fh.setFormatter(fmt)
        log.addHandler(fh)
    if not args.daemonize:
        sh = logging.StreamHandler()
        sh.setFormatter(fmt)
        log.addHandler(sh)

    try:
        pd = Pathod(
            (args.address, args.port),
            craftanchor=args.craftanchor,
            ssl=args.ssl,
            ssloptions=ssloptions,
            staticdir=args.staticdir,
            anchors=args.anchors,
            sizelimit=args.sizelimit,
            noweb=args.noweb,
            nocraft=args.nocraft,
            noapi=args.noapi,
            nohang=args.nohang,
            timeout=args.timeout,
            logreq=args.logreq,
            logresp=args.logresp,
            hexdump=args.hexdump,
            http2_framedump=args.http2_framedump,
            explain=args.explain,
            webdebug=args.webdebug
        )
    except PathodError as v:
        print >> sys.stderr, "Error: %s" % v
        sys.exit(1)
    except language.FileAccessDenied as v:
        print >> sys.stderr, "Error: %s" % v

    if args.daemonize:
        utils.daemonize()

    try:
        print "%s listening on %s:%s" % (
            version.NAMEVERSION,
            pd.address.host,
            pd.address.port
        )
        pd.serve_forever()
    except KeyboardInterrupt:
        pass