http2: add basic protocol handling

netlib/h2/__init__.py

@@ -1 +1,170 @@
 from __future__ import (absolute_import, print_function, division)
+import itertools
+
+from .. import utils
+from .frame import *
+
+
+class HTTP2Protocol(object):
+
+    ERROR_CODES = utils.BiDi(
+        NO_ERROR=0x0,
+        PROTOCOL_ERROR=0x1,
+        INTERNAL_ERROR=0x2,
+        FLOW_CONTROL_ERROR=0x3,
+        SETTINGS_TIMEOUT=0x4,
+        STREAM_CLOSED=0x5,
+        FRAME_SIZE_ERROR=0x6,
+        REFUSED_STREAM=0x7,
+        CANCEL=0x8,
+        COMPRESSION_ERROR=0x9,
+        CONNECT_ERROR=0xa,
+        ENHANCE_YOUR_CALM=0xb,
+        INADEQUATE_SECURITY=0xc,
+        HTTP_1_1_REQUIRED=0xd
+    )
+
+    # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
+    CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'
+
+    ALPN_PROTO_H2 = b'h2'
+
+    HTTP2_DEFAULT_SETTINGS = {
+        SettingsFrame.SETTINGS.SETTINGS_HEADER_TABLE_SIZE: 4096,
+        SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 1,
+        SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: None,
+        SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 2 ** 16 - 1,
+        SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE: 2 ** 14,
+        SettingsFrame.SETTINGS.SETTINGS_MAX_HEADER_LIST_SIZE: None,
+    }
+
+    def __init__(self):
+        self.http2_settings = self.HTTP2_DEFAULT_SETTINGS.copy()
+        self.current_stream_id = None
+        self.encoder = Encoder()
+        self.decoder = Decoder()
+
+    def check_alpn(self):
+        alp = self.get_alpn_proto_negotiated()
+        if alp != self.ALPN_PROTO_H2:
+            raise NotImplementedError(
+                "H2Client can not handle unknown ALP: %s" % alp)
+        print("-> Successfully negotiated 'h2' application layer protocol.")
+
+    def send_connection_preface(self):
+        self.wfile.write(bytes(self.CLIENT_CONNECTION_PREFACE.decode('hex')))
+        self.send_frame(SettingsFrame(state=self))
+
+        frame = Frame.from_file(self.rfile, self)
+        assert isinstance(frame, SettingsFrame)
+        self._apply_settings(frame.settings)
+        self.read_frame()  # read setting ACK frame
+
+        print("-> Connection Preface completed.")
+
+    def next_stream_id(self):
+        if self.current_stream_id is None:
+            self.current_stream_id = 1
+        else:
+            self.current_stream_id += 2
+        return self.current_stream_id
+
+    def send_frame(self, frame):
+        raw_bytes = frame.to_bytes()
+        self.wfile.write(raw_bytes)
+        self.wfile.flush()
+
+    def read_frame(self):
+        frame = Frame.from_file(self.rfile, self)
+        if isinstance(frame, SettingsFrame):
+            self._apply_settings(frame.settings)
+
+        return frame
+
+    def _apply_settings(self, settings):
+        for setting, value in settings.items():
+            old_value = self.http2_settings[setting]
+            if not old_value:
+                old_value = '-'
+
+            self.http2_settings[setting] = value
+            print("-> Setting changed: %s to %d (was %s)" % (
+                SettingsFrame.SETTINGS.get_name(setting),
+                value,
+                str(old_value)))
+
+        self.send_frame(SettingsFrame(state=self, flags=Frame.FLAG_ACK))
+        print("-> New settings acknowledged.")
+
+    def _create_headers(self, headers, stream_id, end_stream=True):
+        # TODO: implement max frame size checks and sending in chunks
+
+        flags = Frame.FLAG_END_HEADERS
+        if end_stream:
+            flags |= Frame.FLAG_END_STREAM
+
+        bytes = HeadersFrame(
+            state=self,
+            flags=flags,
+            stream_id=stream_id,
+            headers=headers).to_bytes()
+        return [bytes]
+
+    def _create_body(self, body, stream_id):
+        if body is None or len(body) == 0:
+            return b''
+
+        # TODO: implement max frame size checks and sending in chunks
+        # TODO: implement flow-control window
+
+        bytes = DataFrame(
+            state=self,
+            flags=Frame.FLAG_END_STREAM,
+            stream_id=stream_id,
+            payload=body).to_bytes()
+        return [bytes]
+
+    def create_request(self, method, path, headers=None, body=None):
+        if headers is None:
+            headers = []
+
+        headers = [
+            (b':method', bytes(method)),
+            (b':path', bytes(path)),
+            (b':scheme', b'https')] + headers
+
+        stream_id = self.next_stream_id()
+
+        return list(itertools.chain(
+            self._create_headers(headers, stream_id, end_stream=(body is None)),
+            self._create_body(body, stream_id)))
+
+    def read_response(self):
+        header_block_fragment = b''
+        body = b''
+
+        while True:
+            frame = self.read_frame()
+            if isinstance(frame, HeadersFrame):
+                header_block_fragment += frame.header_block_fragment
+                if frame.flags | Frame.FLAG_END_HEADERS:
+                    break
+            else:
+                print("Unexpected frame received:")
+                print(frame.human_readable())
+
+        while True:
+            frame = self.read_frame()
+            if isinstance(frame, DataFrame):
+                body += frame.payload
+                if frame.flags | Frame.FLAG_END_STREAM:
+                    break
+            else:
+                print("Unexpected frame received:")
+                print(frame.human_readable())
+
+        headers = {}
+        for header, value in self.decoder.decode(header_block_fragment):
+            headers[header] = value
+
+        return headers[':status'], headers, body

netlib/h2/frame.py

@@ -20,16 +20,24 @@ class Frame(object):
     FLAG_PADDED = 0x8
     FLAG_PRIORITY = 0x20
 
-    def __init__(self, state=None, length=0, flags=FLAG_NO_FLAGS, stream_id=0x0):
+    def __init__(
+            self,
+            state=None,
+            length=0,
+            flags=FLAG_NO_FLAGS,
+            stream_id=0x0):
         valid_flags = reduce(lambda x, y: x | y, self.VALID_FLAGS, 0x0)
         if flags | valid_flags != valid_flags:
             raise ValueError('invalid flags detected.')
 
         if state is None:
+            from . import HTTP2Protocol
+
             class State(object):
                 pass
 
             state = State()
+            state.http2_settings = HTTP2Protocol.HTTP2_DEFAULT_SETTINGS.copy()
             state.encoder = Encoder()
             state.decoder = Decoder()
 
@@ -40,6 +48,14 @@ class Frame(object):
         self.flags = flags
         self.stream_id = stream_id
 
+    def _check_frame_size(self, length):
+        max_length = self.state.http2_settings[
+            SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE]
+        if length > max_length:
+            raise NotImplementedError(
+                "Frame size exceeded: %d, but only %d allowed." % (
+                    length, max_length))
+
     @classmethod
     def from_file(self, fp, state=None):
         """
@@ -54,8 +70,15 @@ class Frame(object):
         flags = fields[3]
         stream_id = fields[4]
 
+        # TODO: check frame size if <= current SETTINGS_MAX_FRAME_SIZE
+
         payload = fp.safe_read(length)
-        return FRAMES[fields[2]].from_bytes(state, length, flags, stream_id, payload)
+        return FRAMES[fields[2]].from_bytes(
+            state,
+            length,
+            flags,
+            stream_id,
+            payload)
 
     @classmethod
     def from_bytes(self, data, state=None):
@@ -64,12 +87,20 @@ class Frame(object):
         # type is already deducted from class
         flags = fields[3]
         stream_id = fields[4]
-        return FRAMES[fields[2]].from_bytes(state, length, flags, stream_id, data[9:])
+
+        return FRAMES[fields[2]].from_bytes(
+            state,
+            length,
+            flags,
+            stream_id,
+            data[9:])
 
     def to_bytes(self):
         payload = self.payload_bytes()
         self.length = len(payload)
 
+        self._check_frame_size(self.length)
+
         b = struct.pack('!HB', self.length & 0xFFFF00, self.length & 0x0000FF)
         b += struct.pack('!B', self.TYPE)
         b += struct.pack('!B', self.flags)
@@ -183,19 +214,20 @@ class HeadersFrame(Frame):
 
         if f.flags & self.FLAG_PADDED:
             f.pad_length = struct.unpack('!B', payload[0])[0]
-            header_block_fragment = payload[1:-f.pad_length]
+            f.header_block_fragment = payload[1:-f.pad_length]
         else:
-            header_block_fragment = payload[0:]
+            f.header_block_fragment = payload[0:]
 
         if f.flags & self.FLAG_PRIORITY:
             f.stream_dependency, f.weight = struct.unpack(
                 '!LB', header_block_fragment[:5])
             f.exclusive = bool(f.stream_dependency >> 31)
             f.stream_dependency &= 0x7FFFFFFF
-            header_block_fragment = header_block_fragment[5:]
+            f.header_block_fragment = f.header_block_fragment[5:]
 
-        for header, value in f.state.decoder.decode(header_block_fragment):
+        # TODO only do this if END_HEADERS or something...
-            f.headers.append((header, value))
+        # for header, value in f.state.decoder.decode(f.header_block_fragment):
+        #     f.headers.append((header, value))
 
         return f
 
@@ -217,6 +249,8 @@ class HeadersFrame(Frame):
                              (int(self.exclusive) << 31) | self.stream_dependency,
                              self.weight)
 
+        # TODO: maybe remove that and only deal with header_block_fragments
+        # inside frames
         b += self.state.encoder.encode(self.headers)
 
         if self.flags & self.FLAG_PADDED:

netlib/h2/h2.py

@@ -1,89 +0,0 @@
-from .. import utils, odict, tcp
-from frame import *
-
-# "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
-CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'
-
-ERROR_CODES = utils.BiDi(
-    NO_ERROR=0x0,
-    PROTOCOL_ERROR=0x1,
-    INTERNAL_ERROR=0x2,
-    FLOW_CONTROL_ERROR=0x3,
-    SETTINGS_TIMEOUT=0x4,
-    STREAM_CLOSED=0x5,
-    FRAME_SIZE_ERROR=0x6,
-    REFUSED_STREAM=0x7,
-    CANCEL=0x8,
-    COMPRESSION_ERROR=0x9,
-    CONNECT_ERROR=0xa,
-    ENHANCE_YOUR_CALM=0xb,
-    INADEQUATE_SECURITY=0xc,
-    HTTP_1_1_REQUIRED=0xd
-)
-
-
-class H2Client(tcp.TCPClient):
-    ALPN_PROTO_H2 = b'h2'
-
-    DEFAULT_SETTINGS = {
-        SettingsFrame.SETTINGS.SETTINGS_HEADER_TABLE_SIZE: 4096,
-        SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 1,
-        SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: None,
-        SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 2 ** 16 - 1,
-        SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE: 2 ** 14,
-        SettingsFrame.SETTINGS.SETTINGS_MAX_HEADER_LIST_SIZE: None,
-    }
-
-    def __init__(self, address, source_address=None):
-        super(H2Client, self).__init__(address, source_address)
-        self.settings = self.DEFAULT_SETTINGS.copy()
-
-    def connect(self, send_preface=True):
-        super(H2Client, self).connect()
-        self.convert_to_ssl(alpn_protos=[self.ALPN_PROTO_H2])
-
-        alp = self.get_alpn_proto_negotiated()
-        if alp != b'h2':
-            raise NotImplementedError(
-                "H2Client can not handle unknown protocol: %s" %
-                alp)
-        print "-> Successfully negotiated 'h2' application layer protocol."
-
-        if send_preface:
-            self.wfile.write(bytes(CLIENT_CONNECTION_PREFACE.decode('hex')))
-            self.send_frame(SettingsFrame())
-
-            frame = Frame.from_file(self.rfile)
-            print frame.human_readable()
-            assert isinstance(frame, SettingsFrame)
-            self.apply_settings(frame.settings)
-
-            print "-> Connection Preface completed."
-
-        print "-> H2Client is ready..."
-
-    def send_frame(self, frame):
-        self.wfile.write(frame.to_bytes())
-        self.wfile.flush()
-
-    def read_frame(self):
-        frame = Frame.from_file(self.rfile)
-        if isinstance(frame, SettingsFrame):
-            self.apply_settings(frame.settings)
-
-        return frame
-
-    def apply_settings(self, settings):
-        for setting, value in settings.items():
-            old_value = self.settings[setting]
-            if not old_value:
-                old_value = '-'
-
-            self.settings[setting] = value
-            print "-> Setting changed: %s to %d (was %s)" %
-                (SettingsFrame.SETTINGS.get_name(setting),
-                 value,
-                 str(old_value))
-
-        self.send_frame(SettingsFrame(flags=Frame.FLAG_ACK))
-        print "-> New settings acknowledged."

test/h2/example.py

@@ -1,18 +0,0 @@
-from netlib.h2.frame import *
-from netlib.h2.h2 import *
-
-c = H2Client(("127.0.0.1", 443))
-c.connect()
-
-c.send_frame(HeadersFrame(
-    flags=(Frame.FLAG_END_HEADERS | Frame.FLAG_END_STREAM),
-    stream_id=0x1,
-    headers=[
-        (b':method', 'GET'),
-        (b':path', b'/index.html'),
-        (b':scheme', b'https'),
-        (b':authority', b'localhost'),
-    ]))
-
-while True:
-    print c.read_frame().human_readable()

test/h2/test_frames.py

@@ -3,6 +3,7 @@ import tutils
 
 from nose.tools import assert_equal
 
+
 def test_invalid_flags():
     tutils.raises(
         ValueError,