Add NAT (postNAT) support to core flow schema, filters, and DNS resolution

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 --- develop ---
 
+* issue#110: Add NAT (postNAT source/destination IP and port) support to the core raw flow schema, filters, and DNS resolution; see flowview_upgrade_nat_columns.php to backfill existing partitions
 * issue#261: Cacti 1.2.x flowview 5.0 update of 16.09 error on call db_check_reconnect inside flow_capture
 * issue#240: Call to db_table_exists is incorrect in some cases
 * issue: Killing processes does not occur when they are in D-State

INFO

@@ -21,7 +21,7 @@
 
 [info]
 name = flowview
-version = 5.0
+version = 5.1
 longname = FlowView
 author = The Cacti Group
 email = developers@cacti.net

arrays.php

@@ -232,9 +232,24 @@
 	'tos'               => 5,
 	'flags'             => 6,
 	'start_time'        => 22,
-	'end_time'          => 21
+	'end_time'          => 21,
+
+	// NAT (Network Address Translation) fields.  Common to Cisco ASA/FTD,
+	// Juniper SRX, and MikroTik RouterOS NetFlow v9/IPFIX exports (issue#110).
+	'post_nat_src_addr'      => 225,
+	'post_nat_src_addr_ipv6' => 281,
+	'post_nat_dst_addr'      => 226,
+	'post_nat_dst_addr_ipv6' => 282,
+	'post_nat_src_port'      => 227,
+	'post_nat_dst_port'      => 228
 ];
 
+// Field IDs kept out of the plain "Supported" bucket in the per-listener
+// template viewer (get_colored_field_column()) even though they are fully
+// extracted/stored -- NAT is an opt-in aggregate extension, not one of the
+// original core fields, so it is called out with its own status/color.
+$flow_fieldids_nat = [225, 226, 227, 228, 281, 282];
+
 $flow_fieldids = array(
 	1 => [
 		'column' => 'dOctets',
@@ -347,6 +362,30 @@
 	160 => [
 		'column' => 'sysuptime',
 		'name'   => 'systemInitTimeMilliseconds'
+	],
+	225 => [
+		'column' => 'post_nat_src_addr',
+		'name'   => 'postNATSourceIPv4Address'
+	],
+	226 => [
+		'column' => 'post_nat_dst_addr',
+		'name'   => 'postNATDestinationIPv4Address'
+	],
+	227 => [
+		'column' => 'post_nat_src_port',
+		'name'   => 'postNAPTSourceTransportPort'
+	],
+	228 => [
+		'column' => 'post_nat_dst_port',
+		'name'   => 'postNAPTDestinationTransportPort'
+	],
+	281 => [
+		'column' => 'post_nat_src_addr',
+		'name'   => 'postNATSourceIPv6Address'
+	],
+	282 => [
+		'column' => 'post_nat_dst_addr',
+		'name'   => 'postNATDestinationIPv6Address'
 	]
 );
 
@@ -726,6 +765,13 @@
 		'default' => 'bar',
 		'array' => $graph_heights
 	),
+	'usenat' => array(
+		'friendly_name' => __('Use NAT Data', 'flowview'),
+		'description' => __('Show the post-NAT (translated) Source/Destination IP and DNS name in report output instead of the original, pre-NAT values.', 'flowview'),
+		'method' => 'checkbox',
+		'value' => '|arg1:usenat|',
+		'default' => ''
+	),
 	'panel_table' => array(
 		'friendly_name' => __('Table Panel', 'flowview'),
 		'description' => __('Should the Table Panel be displayed by default.', 'flowview'),
@@ -847,6 +893,38 @@
 		'max_length' => '20',
 		'size' => '14'
 	),
+	'postnatsourceip' => array(
+		'friendly_name' => __('Post-NAT Source IP', 'flowview'),
+		'description' => __('Filter on the translated (post-NAT) Source IP recorded in the Filter.  This can be a comma delimited list of IPv4 or IPv6 addresses, or a comma delimited list of IPv4 or IPv6 address ranges in CIDR format (eg. 192.168.1.0/24).', 'flowview'),
+		'method' => 'textarea',
+		'value' => '|arg1:postnatsourceip|',
+		'textarea_rows' => '2',
+		'textarea_cols' => '80'
+	),
+	'postnatsourceport' => array(
+		'friendly_name' => __('Post-NAT Source Port', 'flowview'),
+		'description' => __('Filter on the translated (post-NAT) Source Port for in the Filter.  This can be a comma delimited list of Ports.', 'flowview'),
+		'method' => 'textbox',
+		'value' => '|arg1:postnatsourceport|',
+		'max_length' => '20',
+		'size' => '14'
+	),
+	'postnatdestip' => array(
+		'friendly_name' => __('Post-NAT Dest IP', 'flowview'),
+		'description' => __('Filter on the translated (post-NAT) Destination IP recorded in the Filter.  This can be a comma delimited list of IPv4 or IPv6 addresses, or a comma delimited list of IPv4 or IPv6 address ranges in CIDR format (eg. 192.168.1.0/24).', 'flowview'),
+		'method' => 'textarea',
+		'value' => '|arg1:postnatdestip|',
+		'textarea_rows' => '2',
+		'textarea_cols' => '80'
+	),
+	'postnatdestport' => array(
+		'friendly_name' => __('Post-NAT Dest Port', 'flowview'),
+		'description' => __('Filter on the translated (post-NAT) Destination Port for in the Filter.  This can be a comma delimited list of Ports.', 'flowview'),
+		'method' => 'textbox',
+		'value' => '|arg1:postnatdestport|',
+		'max_length' => '20',
+		'size' => '14'
+	),
 	'return' => array(
 		'method' => 'hidden',
 		'value' => get_request_var('return')

flow_collector.php

@@ -1104,7 +1104,17 @@ function process_fv5($p, $ex_addr) {
 			$pps                            . ', ' .
 
 			db_qstr($data['tos'])           . ', ' .
-			db_qstr($data['flags'])         . ')';
+			db_qstr($data['flags'])         . ', ' .
+
+			/* NetFlow v5 has no NAT fields; fill the shared post-NAT columns with defaults */
+			db_qstr('')                     . ', ' .
+			db_qstr('')                     . ', ' .
+			db_qstr('')                     . ', ' .
+			'0'                             . ', ' .
+			db_qstr('')                     . ', ' .
+			db_qstr('')                     . ', ' .
+			db_qstr('')                     . ', ' .
+			'0'                             . ')';
 	}
 
 	if (cacti_sizeof($sql)) {
@@ -1465,7 +1475,7 @@ function get_sql_prefix($flowtime) {
 
 	$last_table = $table;
 
-	return 'INSERT INTO ' . $table . ' (listener_id, template_id, engine_type, engine_id, sampling_interval, ex_addr, sysuptime, src_addr, src_domain, src_rdomain, src_as, src_if, src_prefix, src_port, src_rport, dst_addr, dst_domain, dst_rdomain, dst_as, dst_if, dst_prefix, dst_port, dst_rport, nexthop, protocol, start_time, end_time, flows, packets, bytes, bytes_ppacket, tos, flags) VALUES ';
+	return 'INSERT INTO ' . $table . ' (listener_id, template_id, engine_type, engine_id, sampling_interval, ex_addr, sysuptime, src_addr, src_domain, src_rdomain, src_as, src_if, src_prefix, src_port, src_rport, dst_addr, dst_domain, dst_rdomain, dst_as, dst_if, dst_prefix, dst_port, dst_rport, nexthop, protocol, start_time, end_time, flows, packets, bytes, bytes_ppacket, tos, flags, post_nat_src_addr, post_nat_src_domain, post_nat_src_rdomain, post_nat_src_port, post_nat_dst_addr, post_nat_dst_domain, post_nat_dst_rdomain, post_nat_dst_port) VALUES ';
 }
 
 function process_fv10($p, $ex_addr) {
@@ -1786,6 +1796,30 @@ function process_v9_v10($data, $ex_addr, $flowtime, $fsid, $sysuptime = 0) {
 		return false;
 	}
 
+	/**
+	 * Post-NAT (translated) addresses/ports - issue#110.  Unlike src/dst,
+	 * these are genuinely optional: many vendors (notably Cisco ASA/FTD
+	 * NSEL) split a NAT'd flow across a "Creation" template (which has the
+	 * NAT fields) and a "Teardown" template (which has the byte/packet
+	 * counts but not the NAT fields), so absence here is normal and must
+	 * not fail the record.
+	 */
+	if (isset($data[$flow_fields['post_nat_src_addr_ipv6']])) {
+		$post_nat_src_addr = $data[$flow_fields['post_nat_src_addr_ipv6']];
+	} elseif (isset($data[$flow_fields['post_nat_src_addr']])) {
+		$post_nat_src_addr = $data[$flow_fields['post_nat_src_addr']];
+	} else {
+		$post_nat_src_addr = '';
+	}
+
+	if (isset($data[$flow_fields['post_nat_dst_addr_ipv6']])) {
+		$post_nat_dst_addr = $data[$flow_fields['post_nat_dst_addr_ipv6']];
+	} elseif (isset($data[$flow_fields['post_nat_dst_addr']])) {
+		$post_nat_dst_addr = $data[$flow_fields['post_nat_dst_addr']];
+	} else {
+		$post_nat_dst_addr = '';
+	}
+
 	if (isset($data[$flow_fields['nexthop_ipv6']])) {
 		$nexthop = $data[$flow_fields['nexthop_ipv6']];
 	} elseif (isset($data[$flow_fields['nexthop']])) {
@@ -1848,6 +1882,22 @@ function process_v9_v10($data, $ex_addr, $flowtime, $fsid, $sysuptime = 0) {
 	$dst_domain  = flowview_get_dns_from_ip($dst_addr, 100);
 	$dst_rdomain = flowview_get_rdomain_from_domain($dst_domain, $dst_addr);
 
+	if ($post_nat_src_addr != '') {
+		$post_nat_src_domain  = flowview_get_dns_from_ip($post_nat_src_addr, 100);
+		$post_nat_src_rdomain = flowview_get_rdomain_from_domain($post_nat_src_domain, $post_nat_src_addr);
+	} else {
+		$post_nat_src_domain  = '';
+		$post_nat_src_rdomain = '';
+	}
+
+	if ($post_nat_dst_addr != '') {
+		$post_nat_dst_domain  = flowview_get_dns_from_ip($post_nat_dst_addr, 100);
+		$post_nat_dst_rdomain = flowview_get_rdomain_from_domain($post_nat_dst_domain, $post_nat_dst_addr);
+	} else {
+		$post_nat_dst_domain  = '';
+		$post_nat_dst_rdomain = '';
+	}
+
 	if (isset($data[$flow_fields['src_port']])) {
 		$src_rport = flowview_translate_port($data[$flow_fields['src_port']], false, false);
 	} else {
@@ -1903,7 +1953,17 @@ function process_v9_v10($data, $ex_addr, $flowtime, $fsid, $sysuptime = 0) {
 		check_set($data, $flow_fields['dOctets'])           . ', ' .
 		$pps                                                . ', ' .
 		check_set($data, $flow_fields['tos'])               . ', ' .
-		check_set($data, $flow_fields['flags'])             . ')';
+		check_set($data, $flow_fields['flags'])             . ', ' .
+
+		($post_nat_src_addr != '' ? 'INET6_ATON(' . db_qstr($post_nat_src_addr) . ')':db_qstr('')) . ', ' .
+		db_qstr($post_nat_src_domain)                       . ', ' .
+		db_qstr($post_nat_src_rdomain)                      . ', ' .
+		check_set($data, $flow_fields['post_nat_src_port']) . ', ' .
+
+		($post_nat_dst_addr != '' ? 'INET6_ATON(' . db_qstr($post_nat_dst_addr) . ')':db_qstr('')) . ', ' .
+		db_qstr($post_nat_dst_domain)                       . ', ' .
+		db_qstr($post_nat_dst_rdomain)                      . ', ' .
+		check_set($data, $flow_fields['post_nat_dst_port']) . ')';
 
 	return $sql;
 }

flowview.php

@@ -350,6 +350,7 @@ function load_session_for_filter() {
 					case 'panel_bytes':
 					case 'panel_packets':
 					case 'panel_flows':
+					case 'usenat':
 						if (!isset_request_var($column)) {
 							$column = str_replace('panel_', '', $column);
 
@@ -410,12 +411,17 @@ function flowview_request_vars() {
 			'panel_table',
 			'panel_bytes',
 			'panel_packets',
-			'panel_flows'
+			'panel_flows',
+			'usenat'
 		];
 
 		if (cacti_sizeof($listener)) {
 			foreach($columns as $c) {
-				if (strpos($c, 'panel')) {
+				if ($c == 'usenat') {
+					if (!isset_request_var($c)) {
+						set_request_var($c, $listener[$c] == 'on' ? 'true':'false');
+					}
+				} elseif (strpos($c, 'panel')) {
 					$rv  = str_replace('panel_', '', $c);
 
 					if (!isset_request_var($rv)) {
@@ -484,6 +490,11 @@ function flowview_request_vars() {
 			'options' => array('options' => array('regexp' => '(true|false)')),
 			'default' => 'true'
 		),
+		'usenat' => array(
+			'filter' => FILTER_VALIDATE_REGEXP,
+			'options' => array('options' => array('regexp' => '(true|false)')),
+			'default' => 'false'
+		),
 		'table' => array(
 			'filter' => FILTER_VALIDATE_REGEXP,
 			'options' => array('options' => array('regexp' => '(true|false)')),

flowview_upgrade.php

@@ -598,6 +598,34 @@ function flowview_upgrade($current, $old) {
 		flowview_db_execute('UPDATE plugin_flowview_devices SET bind_address = "0.0.0.0" WHERE bind_address = "0"');
 		flowview_db_execute('UPDATE plugin_flowview_devices SET allowfrom = "0.0.0.0" WHERE allowfrom = "0"');
 
+		if (!flowview_db_column_exists('plugin_flowview_queries', 'postnatsourceip', false)) {
+			cacti_log('Adding post-NAT filter columns to plugin_flowview_queries table.', true, 'FLOWVIEW');
+
+			flowview_db_execute("ALTER TABLE plugin_flowview_queries
+				ADD COLUMN postnatsourceip VARCHAR(255) NOT NULL DEFAULT '' AFTER destas,
+				ADD COLUMN postnatsourceport VARCHAR(255) NOT NULL DEFAULT '' AFTER postnatsourceip,
+				ADD COLUMN postnatdestip VARCHAR(255) NOT NULL DEFAULT '' AFTER postnatsourceport,
+				ADD COLUMN postnatdestport VARCHAR(255) NOT NULL DEFAULT '' AFTER postnatdestip");
+		}
+
+		if (!flowview_db_column_exists('plugin_flowview_queries', 'usenat', false)) {
+			cacti_log('Adding usenat column to plugin_flowview_queries table.', true, 'FLOWVIEW');
+
+			flowview_db_execute("ALTER TABLE plugin_flowview_queries
+				ADD COLUMN usenat CHAR(2) NOT NULL DEFAULT '' AFTER panel_flows");
+		}
+
+		/**
+		 * The post-NAT columns on the (much larger) plugin_flowview_raw_*
+		 * partition tables are intentionally NOT altered here -- a busy
+		 * install can have hundreds of large partitions, and this
+		 * background upgrade needs to stay fast.  Run
+		 * flowview_upgrade_nat_columns.php manually to backfill existing
+		 * partitions; new partitions already include the columns.  See
+		 * flowview_nat_safe_sql() in functions.php for how queries degrade
+		 * gracefully against partitions that haven't been backfilled yet.
+		 */
+
 		cacti_log('Flowview Database Upgrade Complete', true, 'FLOWVIEW');
 	}