Add basic HTTP/2 frame metadata

chains/links/http_meta.py

@@ -2,15 +2,31 @@
 from __future__ import print_function
 
 import dpkt
+import struct
 
 # Local imports
 from chains.links import link
-from chains.utils import file_utils, log_utils, data_utils, compat
+from chains.utils import file_utils, data_utils, compat
 
 
 class HTTPMeta(link.Link):
     """Pull out application meta data from incoming flow data"""
 
+    HTTP2_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'
+    HTTP2_FRAME_TYPES = {
+        0: 'DATA',
+        1: 'HEADERS',
+        2: 'PRIORITY',
+        3: 'RST_STREAM',
+        4: 'SETTINGS',
+        5: 'PUSH_PROMISE',
+        6: 'PING',
+        7: 'GOAWAY',
+        8: 'WINDOW_UPDATE',
+        9: 'CONTINUATION',
+    }
+    HTTP_PORTS = set([80, 8080, 8000, 8888])
+
     def __init__(self):
         """Initialize HTTPMeta Class"""
 
@@ -32,17 +48,19 @@ def http_meta_data(self):
                     request = dpkt.http.Request(flow['payload'])
                     request_data = data_utils.make_dict(request)
                     request_data['uri'] = self._clean_uri(request['uri'])
-                    flow['http'] = {'type':'HTTP_REQUEST', 'data':request_data}
+                    flow['http'] = {'type': 'HTTP_REQUEST', 'data': request_data}
                 except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
-                    flow['http'] = None
+                    http2_data = self._http2_meta(flow)
+                    flow['http'] = {'type': 'HTTP2', 'data': http2_data} if http2_data else None
 
             # Server to Client
             else:
                 try:
                     response = dpkt.http.Response(flow['payload'])
                     flow['http'] = {'type': 'HTTP_RESPONSE', 'data': data_utils.make_dict(response)}
                 except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
-                    flow['http'] = None
+                    http2_data = self._http2_meta(flow)
+                    flow['http'] = {'type': 'HTTP2', 'data': http2_data} if http2_data else None
 
             # Mark non-TCP HTTP
             if flow['http'] and flow['protocol'] != 'TCP':
@@ -56,6 +74,77 @@ def _clean_uri(uri):
         """Clean the URI string"""
         return compat.unquote(uri).replace('+', ' ')
 
+    @classmethod
+    def _http2_meta(cls, flow):
+        """Extract basic cleartext HTTP/2 frame metadata from a flow payload."""
+
+        payload = flow['payload']
+        if not payload:
+            return None
+
+        offset = 0
+        saw_preface = False
+        if payload.startswith(cls.HTTP2_PREFACE):
+            offset = len(cls.HTTP2_PREFACE)
+            saw_preface = True
+        elif not cls._looks_like_http2(flow, payload):
+            return None
+
+        frames = []
+        while offset + 9 <= len(payload):
+            length = cls._read_uint24(payload[offset:offset + 3])
+            frame_type_id = cls._byte_value(payload[offset + 3])
+            flags = cls._byte_value(payload[offset + 4])
+            stream_id = struct.unpack('!I', payload[offset + 5:offset + 9])[0] & 0x7fffffff
+            frame_end = offset + 9 + length
+            if frame_end > len(payload):
+                break
+            frames.append({
+                'type': cls.HTTP2_FRAME_TYPES.get(frame_type_id, 'UNKNOWN'),
+                'type_id': frame_type_id,
+                'flags': flags,
+                'stream_id': stream_id,
+                'length': length,
+            })
+            offset = frame_end
+
+        if not saw_preface and not frames:
+            return None
+
+        return {
+            'preface': saw_preface,
+            'frames': frames,
+            'frame_count': len(frames),
+        }
+
+    @classmethod
+    def _looks_like_http2(cls, flow, payload):
+        """Conservatively decide whether a payload starts with an HTTP/2 frame."""
+
+        if len(payload) < 9:
+            return False
+        if flow.get('sport') not in cls.HTTP_PORTS and flow.get('dport') not in cls.HTTP_PORTS:
+            return False
+
+        frame_type_id = cls._byte_value(payload[3])
+        if frame_type_id not in cls.HTTP2_FRAME_TYPES:
+            return False
+
+        length = cls._read_uint24(payload[0:3])
+        return 9 + length <= len(payload)
+
+    @staticmethod
+    def _byte_value(value):
+        """Return an integer byte value on Python 2 and 3."""
+
+        return value if isinstance(value, int) else ord(value)
+
+    @classmethod
+    def _read_uint24(cls, value):
+        """Read an unsigned 24-bit integer from three network-order bytes."""
+
+        return (cls._byte_value(value[0]) << 16) + (cls._byte_value(value[1]) << 8) + cls._byte_value(value[2])
+
 
 def test():
     """Test for HTTPMeta class"""
@@ -85,5 +174,6 @@ def test():
         if item['http']:
             print('%s %s --> %s  %s' % (item['http']['type'], item['src'], item['dst'], item['http']['data']))
 
+
 if __name__ == '__main__':
     test()

chains/links/http_meta_http2_test.py

@@ -0,0 +1,50 @@
+"""Tests for HTTP/2 metadata extraction."""
+
+from chains.links.http_meta import HTTPMeta
+
+
+def _flow(payload, direction='CTS', sport=12345, dport=80):
+    return {
+        'payload': payload,
+        'direction': direction,
+        'protocol': 'TCP',
+        'sport': sport,
+        'dport': dport,
+    }
+
+
+def _settings_frame():
+    return b'\x00\x00\x00\x04\x00\x00\x00\x00\x00'
+
+
+def test_http2_preface_and_settings_frame_are_reported():
+    http = HTTPMeta()
+    http.input_stream = [_flow(HTTPMeta.HTTP2_PREFACE + _settings_frame())]
+
+    result = list(http.output_stream)[0]
+
+    assert result['http']['type'] == 'HTTP2'
+    assert result['http']['data']['preface'] is True
+    assert result['http']['data']['frame_count'] == 1
+    assert result['http']['data']['frames'][0]['type'] == 'SETTINGS'
+    assert result['http']['data']['frames'][0]['stream_id'] == 0
+
+
+def test_http2_server_settings_frame_is_reported_on_http_port():
+    http = HTTPMeta()
+    http.input_stream = [_flow(_settings_frame(), direction='STC', sport=80, dport=12345)]
+
+    result = list(http.output_stream)[0]
+
+    assert result['http']['type'] == 'HTTP2'
+    assert result['http']['data']['preface'] is False
+    assert result['http']['data']['frames'][0]['type'] == 'SETTINGS'
+
+
+def test_non_http2_payload_is_left_unclassified():
+    http = HTTPMeta()
+    http.input_stream = [_flow(b'not http/2 bytes')]
+
+    result = list(http.output_stream)[0]
+
+    assert result['http'] is None