diff --git a/chains/links/http_meta.py b/chains/links/http_meta.py index 94b787e..06a4017 100644 --- a/chains/links/http_meta.py +++ b/chains/links/http_meta.py @@ -2,15 +2,31 @@ from __future__ import print_function import dpkt +import struct # Local imports from chains.links import link -from chains.utils import file_utils, log_utils, data_utils, compat +from chains.utils import file_utils, data_utils, compat class HTTPMeta(link.Link): """Pull out application meta data from incoming flow data""" + HTTP2_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' + HTTP2_FRAME_TYPES = { + 0: 'DATA', + 1: 'HEADERS', + 2: 'PRIORITY', + 3: 'RST_STREAM', + 4: 'SETTINGS', + 5: 'PUSH_PROMISE', + 6: 'PING', + 7: 'GOAWAY', + 8: 'WINDOW_UPDATE', + 9: 'CONTINUATION', + } + HTTP_PORTS = set([80, 8080, 8000, 8888]) + def __init__(self): """Initialize HTTPMeta Class""" @@ -32,9 +48,10 @@ def http_meta_data(self): request = dpkt.http.Request(flow['payload']) request_data = data_utils.make_dict(request) request_data['uri'] = self._clean_uri(request['uri']) - flow['http'] = {'type':'HTTP_REQUEST', 'data':request_data} + flow['http'] = {'type': 'HTTP_REQUEST', 'data': request_data} except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError): - flow['http'] = None + http2_data = self._http2_meta(flow) + flow['http'] = {'type': 'HTTP2', 'data': http2_data} if http2_data else None # Server to Client else: @@ -42,7 +59,8 @@ def http_meta_data(self): response = dpkt.http.Response(flow['payload']) flow['http'] = {'type': 'HTTP_RESPONSE', 'data': data_utils.make_dict(response)} except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError): - flow['http'] = None + http2_data = self._http2_meta(flow) + flow['http'] = {'type': 'HTTP2', 'data': http2_data} if http2_data else None # Mark non-TCP HTTP if flow['http'] and flow['protocol'] != 'TCP': @@ -56,6 +74,77 @@ def _clean_uri(uri): """Clean the URI string""" return compat.unquote(uri).replace('+', ' ') + @classmethod + def _http2_meta(cls, flow): + """Extract basic cleartext HTTP/2 frame metadata from a flow payload.""" + + payload = flow['payload'] + if not payload: + return None + + offset = 0 + saw_preface = False + if payload.startswith(cls.HTTP2_PREFACE): + offset = len(cls.HTTP2_PREFACE) + saw_preface = True + elif not cls._looks_like_http2(flow, payload): + return None + + frames = [] + while offset + 9 <= len(payload): + length = cls._read_uint24(payload[offset:offset + 3]) + frame_type_id = cls._byte_value(payload[offset + 3]) + flags = cls._byte_value(payload[offset + 4]) + stream_id = struct.unpack('!I', payload[offset + 5:offset + 9])[0] & 0x7fffffff + frame_end = offset + 9 + length + if frame_end > len(payload): + break + frames.append({ + 'type': cls.HTTP2_FRAME_TYPES.get(frame_type_id, 'UNKNOWN'), + 'type_id': frame_type_id, + 'flags': flags, + 'stream_id': stream_id, + 'length': length, + }) + offset = frame_end + + if not saw_preface and not frames: + return None + + return { + 'preface': saw_preface, + 'frames': frames, + 'frame_count': len(frames), + } + + @classmethod + def _looks_like_http2(cls, flow, payload): + """Conservatively decide whether a payload starts with an HTTP/2 frame.""" + + if len(payload) < 9: + return False + if flow.get('sport') not in cls.HTTP_PORTS and flow.get('dport') not in cls.HTTP_PORTS: + return False + + frame_type_id = cls._byte_value(payload[3]) + if frame_type_id not in cls.HTTP2_FRAME_TYPES: + return False + + length = cls._read_uint24(payload[0:3]) + return 9 + length <= len(payload) + + @staticmethod + def _byte_value(value): + """Return an integer byte value on Python 2 and 3.""" + + return value if isinstance(value, int) else ord(value) + + @classmethod + def _read_uint24(cls, value): + """Read an unsigned 24-bit integer from three network-order bytes.""" + + return (cls._byte_value(value[0]) << 16) + (cls._byte_value(value[1]) << 8) + cls._byte_value(value[2]) + def test(): """Test for HTTPMeta class""" @@ -85,5 +174,6 @@ def test(): if item['http']: print('%s %s --> %s %s' % (item['http']['type'], item['src'], item['dst'], item['http']['data'])) + if __name__ == '__main__': test() diff --git a/chains/links/http_meta_http2_test.py b/chains/links/http_meta_http2_test.py new file mode 100644 index 0000000..9f11d38 --- /dev/null +++ b/chains/links/http_meta_http2_test.py @@ -0,0 +1,50 @@ +"""Tests for HTTP/2 metadata extraction.""" + +from chains.links.http_meta import HTTPMeta + + +def _flow(payload, direction='CTS', sport=12345, dport=80): + return { + 'payload': payload, + 'direction': direction, + 'protocol': 'TCP', + 'sport': sport, + 'dport': dport, + } + + +def _settings_frame(): + return b'\x00\x00\x00\x04\x00\x00\x00\x00\x00' + + +def test_http2_preface_and_settings_frame_are_reported(): + http = HTTPMeta() + http.input_stream = [_flow(HTTPMeta.HTTP2_PREFACE + _settings_frame())] + + result = list(http.output_stream)[0] + + assert result['http']['type'] == 'HTTP2' + assert result['http']['data']['preface'] is True + assert result['http']['data']['frame_count'] == 1 + assert result['http']['data']['frames'][0]['type'] == 'SETTINGS' + assert result['http']['data']['frames'][0]['stream_id'] == 0 + + +def test_http2_server_settings_frame_is_reported_on_http_port(): + http = HTTPMeta() + http.input_stream = [_flow(_settings_frame(), direction='STC', sport=80, dport=12345)] + + result = list(http.output_stream)[0] + + assert result['http']['type'] == 'HTTP2' + assert result['http']['data']['preface'] is False + assert result['http']['data']['frames'][0]['type'] == 'SETTINGS' + + +def test_non_http2_payload_is_left_unclassified(): + http = HTTPMeta() + http.input_stream = [_flow(b'not http/2 bytes')] + + result = list(http.output_stream)[0] + + assert result['http'] is None