mitmproxy/pathod/language/http2.py

import pyparsing as pp
from mitmproxy.net import http
from mitmproxy.net.http import user_agents, Headers
from . import base, message


"""
    Normal HTTP requests:
        <method>:<path>:<header>:<body>
    e.g.:
        GET:/
        GET:/:h"foo"="bar"
        POST:/:h"foo"="bar":b'content body payload'

    Normal HTTP responses:
        <code>:<header>:<body>
    e.g.:
        200
        302:h"foo"="bar"
        404:h"foo"="bar":b'content body payload'

    Individual HTTP/2 frames:
        h2f:<payload_length>:<type>:<flags>:<stream_id>:<payload>
    e.g.:
        h2f:0:PING
        h2f:42:HEADERS:END_HEADERS:0x1234567:foo=bar,host=example.com
        h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload'
"""


def get_header(val, headers):
    """
        Header keys may be Values, so we have to "generate" them as we try the
        match.
    """
    for h in headers:
        k = h.key.get_generator({})
        if len(k) == len(val) and k[:].lower() == val.lower():
            return h
    return None


class _HeaderMixin:
    unique_name = None  # type: ignore

    def values(self, settings):
        return (
            self.key.get_generator(settings),
            self.value.get_generator(settings),
        )


class _HTTP2Message(message.Message):
    @property
    def actions(self):
        return []  # self.toks(actions._Action)

    @property
    def headers(self):
        headers = self.toks(_HeaderMixin)

        if not self.raw:
            if not get_header(b"content-length", headers):
                if not self.body:
                    length = 0
                else:
                    length = len(self.body.string())
                headers.append(
                    Header(
                        base.TokValueLiteral("content-length"),
                        base.TokValueLiteral(str(length)),
                    )
                )
        return headers

    @property
    def raw(self):
        return bool(self.tok(Raw))

    @property
    def body(self):
        return self.tok(Body)

    def resolve(self, settings):
        return self


class StatusCode(base.Integer):
    pass


class Method(base.OptionsOrValue):
    options = [
        "GET",
        "HEAD",
        "POST",
        "PUT",
        "DELETE",
    ]


class Path(base.Value):
    pass


class Header(_HeaderMixin, base.KeyValue):
    preamble = "h"


class ShortcutContentType(_HeaderMixin, base.Value):
    preamble = "c"
    key = base.TokValueLiteral("content-type")


class ShortcutLocation(_HeaderMixin, base.Value):
    preamble = "l"
    key = base.TokValueLiteral("location")


class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):
    preamble = "u"
    options = [i[1] for i in user_agents.UASTRINGS]
    key = base.TokValueLiteral("user-agent")

    def values(self, settings):
        value = self.value.val
        if self.option_used:
            value = user_agents.get_by_shortcut(value.lower().decode())[2].encode()

        return (
            self.key.get_generator(settings),
            value
        )


class Raw(base.CaselessLiteral):
    TOK = "r"


class Body(base.Value):
    preamble = "b"


class Times(base.Integer):
    preamble = "x"


class Response(_HTTP2Message):
    unique_name = None  # type: ignore
    comps = (
        Header,
        Body,
        ShortcutContentType,
        ShortcutLocation,
        Raw,
    )

    def __init__(self, tokens):
        super(Response, self).__init__(tokens)
        self.rendered_values = None
        self.stream_id = 2

    @property
    def status_code(self):
        return self.tok(StatusCode)

    @classmethod
    def expr(cls):
        parts = [i.expr() for i in cls.comps]
        atom = pp.MatchFirst(parts)
        resp = pp.And(
            [
                StatusCode.expr(),
                pp.ZeroOrMore(base.Sep + atom)
            ]
        )
        resp = resp.setParseAction(cls)
        return resp

    def values(self, settings):
        if self.rendered_values:
            return self.rendered_values
        else:
            headers = Headers([header.values(settings) for header in self.headers])

            body = self.body
            if body:
                body = body.string()

            resp = http.Response(
                b'HTTP/2.0',
                int(self.status_code.string()),
                b'',
                headers,
                body,
            )
            resp.stream_id = self.stream_id

            self.rendered_values = settings.protocol.assemble(resp)
            return self.rendered_values

    def spec(self):
        return ":".join([i.spec() for i in self.tokens])


class NestedResponse(base.NestedMessage):
    preamble = "s"
    nest_type = Response


class Request(_HTTP2Message):
    comps = (
        Header,
        ShortcutContentType,
        ShortcutUserAgent,
        Raw,
        NestedResponse,
        Body,
        Times,
    )
    logattrs = ["method", "path"]

    def __init__(self, tokens):
        super(Request, self).__init__(tokens)
        self.rendered_values = None
        self.stream_id = 1

    @property
    def method(self):
        return self.tok(Method)

    @property
    def path(self):
        return self.tok(Path)

    @property
    def nested_response(self):
        return self.tok(NestedResponse)

    @property
    def times(self):
        return self.tok(Times)

    @classmethod
    def expr(cls):
        parts = [i.expr() for i in cls.comps]
        atom = pp.MatchFirst(parts)
        resp = pp.And(
            [
                Method.expr(),
                base.Sep,
                Path.expr(),
                pp.ZeroOrMore(base.Sep + atom)
            ]
        )
        resp = resp.setParseAction(cls)
        return resp

    def values(self, settings):
        if self.rendered_values:
            return self.rendered_values
        else:
            path = self.path.string()
            if self.nested_response:
                path += self.nested_response.parsed.spec().encode()

            headers = Headers([header.values(settings) for header in self.headers])

            body = self.body
            if body:
                body = body.string()

            req = http.Request(
                b'',
                self.method.string(),
                b'http',
                b'',
                b'',
                path,
                (2, 0),
                headers,
                body,
            )
            req.stream_id = self.stream_id

            self.rendered_values = settings.protocol.assemble(req)
            return self.rendered_values

    def spec(self):
        return ":".join([i.spec() for i in self.tokens])


def make_error_response(reason, body=None):
    tokens = [
        StatusCode("800"),
        Body(base.TokValueLiteral("pathod error: " + (body or reason))),
    ]
    return Response(tokens)
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`import pyparsing as pp`
The final piece: netlib -> mitproxy.net 2016-10-19 22:56:38 +00:00			`from mitmproxy.net import http`
			`from mitmproxy.net.http import user_agents, Headers`
fix prospector code smells 2015-06-18 16:05:09 +00:00			`from . import base, message`
http2: add request-response handling 2015-06-08 08:45:17 +00:00
Extend Mypy checking to pathod * mypy checking pathod * initial commit , fixed errors * tox: mypy checking to pathod * Fixed mypy test failed * issue was with args in custom_contentview.py * tox: mypy checking to #2221 * follow-import=skip since we cant provide args to custom_contentview.py during mypy testing * Lint , Typo Fixed * code style: module import 2017-05-01 23:49:25 +00:00
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`"""`
			`Normal HTTP requests:`
			`<method>:<path>:<header>:<body>`
			`e.g.:`
			`GET:/`
fix prospector code smells 2015-06-18 09:07:33 +00:00			`GET:/:h"foo"="bar"`
			`POST:/:h"foo"="bar":b'content body payload'`

			`Normal HTTP responses:`
			`<code>:<header>:<body>`
			`e.g.:`
			`200`
			`302:h"foo"="bar"`
			`404:h"foo"="bar":b'content body payload'`
http2: add request-response handling 2015-06-08 08:45:17 +00:00
			`Individual HTTP/2 frames:`
			`h2f:<payload_length>:<type>:<flags>:<stream_id>:<payload>`
			`e.g.:`
			`h2f:0:PING`
			`h2f:42:HEADERS:END_HEADERS:0x1234567:foo=bar,host=example.com`
			`h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload'`
			`"""`

pathod: fix most flake8 offenses 2016-05-28 20:25:54 +00:00
http2: implement more language features 2015-06-22 10:42:39 +00:00			`def get_header(val, headers):`
			`"""`
			`Header keys may be Values, so we have to "generate" them as we try the`
			`match.`
			`"""`
			`for h in headers:`
			`k = h.key.get_generator({})`
			`if len(k) == len(val) and k[:].lower() == val.lower():`
			`return h`
			`return None`


python3: clean up class brackets 2016-10-17 04:29:45 +00:00			`class _HeaderMixin:`
Extend Mypy checking to pathod * mypy checking pathod * initial commit , fixed errors * tox: mypy checking to pathod * Fixed mypy test failed * issue was with args in custom_contentview.py * tox: mypy checking to #2221 * follow-import=skip since we cant provide args to custom_contentview.py during mypy testing * Lint , Typo Fixed * code style: module import 2017-05-01 23:49:25 +00:00			`unique_name = None # type: ignore`
http2: implement more language features 2015-06-22 10:42:39 +00:00
			`def values(self, settings):`
			`return (`
			`self.key.get_generator(settings),`
			`self.value.get_generator(settings),`
			`)`

pathod: fix most flake8 offenses 2016-05-28 20:25:54 +00:00
http2: implement more language features 2015-06-22 10:42:39 +00:00			`class _HTTP2Message(message.Message):`
			`@property`
			`def actions(self):`
			`return [] # self.toks(actions._Action)`

			`@property`
			`def headers(self):`
			`headers = self.toks(_HeaderMixin)`

			`if not self.raw:`
Py3: Find content-length header by bytes 2016-06-08 10:58:39 +00:00			`if not get_header(b"content-length", headers):`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`if not self.body:`
			`length = 0`
			`else:`
			`length = len(self.body.string())`
			`headers.append(`
			`Header(`
			`base.TokValueLiteral("content-length"),`
			`base.TokValueLiteral(str(length)),`
			`)`
			`)`
			`return headers`

			`@property`
			`def raw(self):`
			`return bool(self.tok(Raw))`

			`@property`
			`def body(self):`
			`return self.tok(Body)`

			`def resolve(self, settings):`
			`return self`


code -> status_code, httpversion -> http_version 2015-09-17 13:25:28 +00:00			`class StatusCode(base.Integer):`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`pass`

http2: add request-response handling 2015-06-08 08:45:17 +00:00
			`class Method(base.OptionsOrValue):`
			`options = [`
			`"GET",`
			`"HEAD",`
			`"POST",`
			`"PUT",`
			`"DELETE",`
			`]`


			`class Path(base.Value):`
			`pass`


http2: implement more language features 2015-06-22 10:42:39 +00:00			`class Header(_HeaderMixin, base.KeyValue):`
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`preamble = "h"`

http2: implement more language features 2015-06-22 10:42:39 +00:00
			`class ShortcutContentType(_HeaderMixin, base.Value):`
			`preamble = "c"`
			`key = base.TokValueLiteral("content-type")`


			`class ShortcutLocation(_HeaderMixin, base.Value):`
			`preamble = "l"`
			`key = base.TokValueLiteral("location")`


			`class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):`
			`preamble = "u"`
use new netlib module names 2015-07-15 20:04:25 +00:00			`options = [i[1] for i in user_agents.UASTRINGS]`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`key = base.TokValueLiteral("user-agent")`

http2: implement Headers for request & response improve test coverage fix super ctor call fix legacy httpversion simpliy SSLInfo without ALPN 2015-06-16 11:52:41 +00:00			`def values(self, settings):`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`value = self.value.val`
			`if self.option_used:`
Py3: Properly encode() access to user_agents 2016-06-07 06:37:46 +00:00			`value = user_agents.get_by_shortcut(value.lower().decode())[2].encode()`
http2: implement more language features 2015-06-22 10:42:39 +00:00
http2: implement Headers for request & response improve test coverage fix super ctor call fix legacy httpversion simpliy SSLInfo without ALPN 2015-06-16 11:52:41 +00:00			`return (`
			`self.key.get_generator(settings),`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`value`
http2: implement Headers for request & response improve test coverage fix super ctor call fix legacy httpversion simpliy SSLInfo without ALPN 2015-06-16 11:52:41 +00:00			`)`

http2: add request-response handling 2015-06-08 08:45:17 +00:00
http2: implement more language features 2015-06-22 10:42:39 +00:00			`class Raw(base.CaselessLiteral):`
			`TOK = "r"`


http2: add request-response handling 2015-06-08 08:45:17 +00:00			`class Body(base.Value):`
			`preamble = "b"`


			`class Times(base.Integer):`
			`preamble = "x"`

http2: implement Headers for request & response improve test coverage fix super ctor call fix legacy httpversion simpliy SSLInfo without ALPN 2015-06-16 11:52:41 +00:00
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`class Response(_HTTP2Message):`
Extend Mypy checking to pathod * mypy checking pathod * initial commit , fixed errors * tox: mypy checking to pathod * Fixed mypy test failed * issue was with args in custom_contentview.py * tox: mypy checking to #2221 * follow-import=skip since we cant provide args to custom_contentview.py during mypy testing * Lint , Typo Fixed * code style: module import 2017-05-01 23:49:25 +00:00			`unique_name = None # type: ignore`
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`comps = (`
			`Header,`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`Body,`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`ShortcutContentType,`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`ShortcutLocation,`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`Raw,`
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`)`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00
			`def __init__(self, tokens):`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`super(Response, self).__init__(tokens)`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`self.rendered_values = None`
use netlib http semantics 2015-07-29 09:26:10 +00:00			`self.stream_id = 2`
http2: add request-response handling 2015-06-08 08:45:17 +00:00
			`@property`
code -> status_code, httpversion -> http_version 2015-09-17 13:25:28 +00:00			`def status_code(self):`
			`return self.tok(StatusCode)`
http2: add request-response handling 2015-06-08 08:45:17 +00:00
			`@classmethod`
fix prospector code smells 2015-06-18 09:07:33 +00:00			`def expr(cls):`
			`parts = [i.expr() for i in cls.comps]`
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`atom = pp.MatchFirst(parts)`
			`resp = pp.And(`
			`[`
code -> status_code, httpversion -> http_version 2015-09-17 13:25:28 +00:00			`StatusCode.expr(),`
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`pp.ZeroOrMore(base.Sep + atom)`
			`]`
			`)`
fix prospector code smells 2015-06-18 09:07:33 +00:00			`resp = resp.setParseAction(cls)`
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`return resp`

http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`def values(self, settings):`
			`if self.rendered_values:`
			`return self.rendered_values`
			`else:`
adjust to new netlib headers 2015-09-05 16:16:08 +00:00			`headers = Headers([header.values(settings) for header in self.headers])`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00
			`body = self.body`
			`if body:`
			`body = body.string()`

adjust to netlib changes 2015-09-16 16:44:34 +00:00			`resp = http.Response(`
Py3: fix http2 bytes issue 2016-06-13 07:32:13 +00:00			`b'HTTP/2.0',`
fix pathod http2 response creation 2016-09-22 03:36:26 +00:00			`int(self.status_code.string()),`
Py3: Pass bytes to http Request and Response 2016-06-07 08:10:39 +00:00			`b'',`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`headers,`
use netlib http semantics 2015-07-29 09:26:10 +00:00			`body,`
			`)`
			`resp.stream_id = self.stream_id`

			`self.rendered_values = settings.protocol.assemble(resp)`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`return self.rendered_values`

			`def spec(self):`
			`return ":".join([i.spec() for i in self.tokens])`


http2: implement embedded response 2015-06-22 14:11:55 +00:00			`class NestedResponse(base.NestedMessage):`
			`preamble = "s"`
			`nest_type = Response`


			`class Request(_HTTP2Message):`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`comps = (`
			`Header,`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`ShortcutContentType,`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`ShortcutUserAgent,`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`Raw,`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`NestedResponse,`
			`Body,`
			`Times,`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`)`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`logattrs = ["method", "path"]`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00
			`def __init__(self, tokens):`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`super(Request, self).__init__(tokens)`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`self.rendered_values = None`
use netlib http semantics 2015-07-29 09:26:10 +00:00			`self.stream_id = 1`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00
			`@property`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`def method(self):`
			`return self.tok(Method)`

			`@property`
			`def path(self):`
			`return self.tok(Path)`

			`@property`
			`def nested_response(self):`
			`return self.tok(NestedResponse)`

			`@property`
			`def times(self):`
			`return self.tok(Times)`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00
			`@classmethod`
fix prospector code smells 2015-06-18 09:07:33 +00:00			`def expr(cls):`
			`parts = [i.expr() for i in cls.comps]`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`atom = pp.MatchFirst(parts)`
			`resp = pp.And(`
			`[`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`Method.expr(),`
			`base.Sep,`
			`Path.expr(),`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`pp.ZeroOrMore(base.Sep + atom)`
			`]`
http2: add request-response handling 2015-06-08 08:45:17 +00:00			`)`
fix prospector code smells 2015-06-18 09:07:33 +00:00			`resp = resp.setParseAction(cls)`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`return resp`
http2: add request-response handling 2015-06-08 08:45:17 +00:00
			`def values(self, settings):`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`if self.rendered_values:`
			`return self.rendered_values`
			`else:`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`path = self.path.string()`
			`if self.nested_response:`
Py3: encode() the spec before appending to path 2016-06-08 10:59:21 +00:00			`path += self.nested_response.parsed.spec().encode()`
http2: implement embedded response 2015-06-22 14:11:55 +00:00
adjust to new netlib headers 2015-09-05 16:16:08 +00:00			`headers = Headers([header.values(settings) for header in self.headers])`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00
			`body = self.body`
			`if body:`
http2: implement error response 2015-06-16 09:33:10 +00:00			`body = body.string()`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00
adjust to netlib changes 2015-09-16 16:44:34 +00:00			`req = http.Request(`
Py3: Pass bytes to http Request and Response 2016-06-07 08:10:39 +00:00			`b'',`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`self.method.string(),`
minor fixes 2016-07-02 00:49:18 +00:00			`b'http',`
Py3: Pass bytes to http Request and Response 2016-06-07 08:10:39 +00:00			`b'',`
			`b'',`
http2: implement embedded response 2015-06-22 14:11:55 +00:00			`path,`
use netlib http semantics 2015-07-29 09:26:10 +00:00			`(2, 0),`
http2: implement more language features 2015-06-22 10:42:39 +00:00			`headers,`
use netlib http semantics 2015-07-29 09:26:10 +00:00			`body,`
			`)`
			`req.stream_id = self.stream_id`

			`self.rendered_values = settings.protocol.assemble(req)`
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`return self.rendered_values`
http2: add request-response handling 2015-06-08 08:45:17 +00:00
			`def spec(self):`
			`return ":".join([i.spec() for i in self.tokens])`

pathod: fix most flake8 offenses 2016-05-28 20:25:54 +00:00
http2: add request-response to pathod 2015-06-11 14:13:22 +00:00			`def make_error_response(reason, body=None):`
http2: implement error response 2015-06-16 09:33:10 +00:00			`tokens = [`
code -> status_code, httpversion -> http_version 2015-09-17 13:25:28 +00:00			`StatusCode("800"),`
http2: implement error response 2015-06-16 09:33:10 +00:00			`Body(base.TokValueLiteral("pathod error: " + (body or reason))),`
			`]`
			`return Response(tokens)`