TLS: export heuristic fingerprint as metadata (#2609)

example/ndpiReader.c

@@ -228,6 +228,7 @@ struct receiver *receivers = NULL, *topReceivers = NULL;
 
 #define WIRESHARK_METADATA_SERVERNAME	0x01
 #define WIRESHARK_METADATA_JA4C		0x02
+#define WIRESHARK_METADATA_TLS_HEURISTICS_MATCHING_FINGERPRINT	0x03
 
 struct ndpi_packet_tlv {
   u_int16_t type;
@@ -4674,6 +4675,22 @@ static void ndpi_process_packet(u_char *args,
         tot_len += 4 + htons(tlv->length);
         tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
       }
+      if(flow->ssh_tls.obfuscated_heur_matching_set.pkts[0] != 0) {
+        tlv->type = ntohs(WIRESHARK_METADATA_TLS_HEURISTICS_MATCHING_FINGERPRINT);
+        tlv->length = ntohs(sizeof(struct ndpi_tls_obfuscated_heuristic_matching_set));
+        struct ndpi_tls_obfuscated_heuristic_matching_set *s =  (struct ndpi_tls_obfuscated_heuristic_matching_set *)tlv->data;
+        s->bytes[0] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[0]);
+        s->bytes[1] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[1]);
+        s->bytes[2] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[2]);
+        s->bytes[3] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.bytes[3]);
+        s->pkts[0] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[0]);
+        s->pkts[1] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[1]);
+        s->pkts[2] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[2]);
+        s->pkts[3] = ntohl(flow->ssh_tls.obfuscated_heur_matching_set.pkts[3]);
+        /* TODO: boundary check */
+        tot_len += 4 + htons(tlv->length);
+        tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
+      }
 
       flow->detection_completed = 2; /* Avoid exporting metadata again.
                                         If we really want to have the metadata on Wireshark for *all*

example/reader_util.c

@@ -1583,6 +1583,10 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
     }
   }
 
+  if(flow->ndpi_flow->tls_quic.obfuscated_heur_state && flow->ndpi_flow->tls_quic.obfuscated_heur_matching_set)
+    memcpy(&flow->ssh_tls.obfuscated_heur_matching_set, flow->ndpi_flow->tls_quic.obfuscated_heur_matching_set,
+           sizeof(struct ndpi_tls_obfuscated_heuristic_matching_set));
+
   if(!monitoring_enabled) {
     add_to_address_port_list(&flow->stun.mapped_address, &flow->ndpi_flow->stun.mapped_address);
     add_to_address_port_list(&flow->stun.peer_address, &flow->ndpi_flow->stun.peer_address);

example/reader_util.h

@@ -302,6 +302,8 @@ typedef struct ndpi_flow_info {
     ndpi_cipher_weakness client_unsafe_cipher, server_unsafe_cipher;
 
     u_int32_t quic_version;
+
+    struct ndpi_tls_obfuscated_heuristic_matching_set obfuscated_heur_matching_set;
   } ssh_tls;
 
   struct {

src/include/ndpi_typedefs.h

@@ -1260,6 +1260,11 @@ struct os_fingerprint {
   enum operating_system_hint os;
 };
 
+struct ndpi_tls_obfuscated_heuristic_matching_set {
+  u_int32_t bytes[4];
+  u_int32_t pkts[4];
+};
+
 struct ndpi_flow_struct {
   u_int16_t detected_protocol_stack[NDPI_PROTOCOL_SIZE];
 
@@ -1373,6 +1378,7 @@ struct ndpi_flow_struct {
     message_t message[2]; /* Directions */
     u_int8_t certificate_processed:1, change_cipher_from_client:1, change_cipher_from_server:1, from_opportunistic_tls:1, pad:4;
     struct tls_obfuscated_heuristic_state *obfuscated_heur_state;
+    struct ndpi_tls_obfuscated_heuristic_matching_set *obfuscated_heur_matching_set;
   } tls_quic; /* Used also by DTLS and POPS/IMAPS/SMTPS/FTPS */
 
   union {
@@ -1608,8 +1614,8 @@ struct ndpi_flow_struct {
 _Static_assert(sizeof(((struct ndpi_flow_struct *)0)->protos) <= 264,
                "Size of the struct member protocols increased to more than 264 bytes, "
                "please check if this change is necessary.");
-_Static_assert(sizeof(struct ndpi_flow_struct) <= 1192,
-               "Size of the flow struct increased to more than 1192 bytes, "
+_Static_assert(sizeof(struct ndpi_flow_struct) <= 1200,
+               "Size of the flow struct increased to more than 1200 bytes, "
                "please check if this change is necessary.");
 #endif
 #endif

src/lib/ndpi_main.c

@@ -6817,6 +6817,8 @@ void ndpi_free_flow_data(struct ndpi_flow_struct* flow) {
 
     if(flow->tls_quic.obfuscated_heur_state)
       ndpi_free(flow->tls_quic.obfuscated_heur_state);
+    if(flow->tls_quic.obfuscated_heur_matching_set)
+      ndpi_free(flow->tls_quic.obfuscated_heur_matching_set);
   }
 }
 

src/lib/protocols/tls.c

@@ -380,6 +380,20 @@ static int tls_obfuscated_heur_search(struct ndpi_detection_module_struct* ndpi_
       NDPI_LOG_DBG2(ndpi_struct, "TLS-Obf-Heur: set %d completed\n", i);
       if(check_set(ndpi_struct, set)) {
         /* Heuristic match */
+
+        /* Export the matching set as metadata */
+        flow->tls_quic.obfuscated_heur_matching_set = ndpi_calloc(1, sizeof(struct ndpi_tls_obfuscated_heuristic_matching_set));
+        if(flow->tls_quic.obfuscated_heur_matching_set) {
+          flow->tls_quic.obfuscated_heur_matching_set->bytes[0] = set->bytes[0];
+          flow->tls_quic.obfuscated_heur_matching_set->bytes[1] = set->bytes[1];
+          flow->tls_quic.obfuscated_heur_matching_set->bytes[2] = set->bytes[2];
+          flow->tls_quic.obfuscated_heur_matching_set->bytes[3] = set->bytes[3];
+          flow->tls_quic.obfuscated_heur_matching_set->pkts[0] = set->pkts[0];
+          flow->tls_quic.obfuscated_heur_matching_set->pkts[1] = set->pkts[1];
+          flow->tls_quic.obfuscated_heur_matching_set->pkts[2] = set->pkts[2];
+          flow->tls_quic.obfuscated_heur_matching_set->pkts[3] = set->pkts[3];
+        }
+
         return 2; /* Found */
       } else {
         /* Close this set and open a new one... */

wireshark/ndpi.lua

@@ -70,6 +70,7 @@ local mtd_types = {
    [0] = "Padding",
    [1] = "Server Name",
    [2] = "JA4C"
+   [3] = "TLS Heuristic Fingerprint",
 }
 ndpi_fds.metadata_type        = ProtoField.new("nDPI Metadata Type", "ndpi.metadata.type", ftypes.UINT16, mtd_types)
 ndpi_fds.metadata_length      = ProtoField.new("nDPI Metadata Length", "ndpi.metadata.length", ftypes.UINT16)
@@ -78,6 +79,16 @@ ndpi_fds.metadata_value       = ProtoField.new("nDPI Metadata Value", "ndpi.meta
 -- Specific fields
 ndpi_fds.metadata_server_name = ProtoField.new("nDPI Server Name", "ndpi.metadata.server_name", ftypes.STRING)
 ndpi_fds.metadata_ja4c        = ProtoField.new("nDPI JA4C", "ndpi.metadata.ja4c", ftypes.STRING)
+ndpi_fds.metadata             = ProtoField.new("nDPI Metadata", "ndpi.metadata", ftypes.NONE)
+ndpi_fds.metadata_tls_heuristic_fingerprint = ProtoField.new("nDPI TLS Heuristic Fingerprint", "ndpi.metadata.tls_heuristic_fingerprint", ftypes.NONE)
+ndpi_fds.metadata_tls_heuristic_fingerprint_bytes0 = ProtoField.new("Bytes[0]", "ndpi.metadata.tls_heuristic_fingerprint.bytes0", ftypes.UINT32)
+ndpi_fds.metadata_tls_heuristic_fingerprint_bytes1 = ProtoField.new("Bytes[1]", "ndpi.metadata.tls_heuristic_fingerprint.bytes1", ftypes.UINT32)
+ndpi_fds.metadata_tls_heuristic_fingerprint_bytes2 = ProtoField.new("Bytes[2]", "ndpi.metadata.tls_heuristic_fingerprint.bytes2", ftypes.UINT32)
+ndpi_fds.metadata_tls_heuristic_fingerprint_bytes3 = ProtoField.new("Bytes[3]", "ndpi.metadata.tls_heuristic_fingerprint.bytes3", ftypes.UINT32)
+ndpi_fds.metadata_tls_heuristic_fingerprint_pkts0 = ProtoField.new("Pkts[0]", "ndpi.metadata.tls_heuristic_fingerprint.pkts0", ftypes.UINT32)
+ndpi_fds.metadata_tls_heuristic_fingerprint_pkts1 = ProtoField.new("Pkts[1]", "ndpi.metadata.tls_heuristic_fingerprint.pkts1", ftypes.UINT32)
+ndpi_fds.metadata_tls_heuristic_fingerprint_pkts2 = ProtoField.new("Pkts[2]", "ndpi.metadata.tls_heuristic_fingerprint.pkts2", ftypes.UINT32)
+ndpi_fds.metadata_tls_heuristic_fingerprint_pkts3 = ProtoField.new("Pkts[3]", "ndpi.metadata.tls_heuristic_fingerprint.pkts3", ftypes.UINT32)
 
 
 local flow_risks = {}
@@ -1965,8 +1976,19 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
 		  metadata_tree:append_text(" ServerName: " .. trailer_tvb(offset + 4, mtd_length):string())
 		  metadata_tree:add(ndpi_fds.metadata_server_name, trailer_tvb(offset + 4, mtd_length))
 	       elseif mtd_type == 2 then
-		  metadata_tree:append_text(" JA4C: " .. trailer_tvb(offset + 4, mtd_length):string())
-		  metadata_tree:add(ndpi_fds.metadata_ja4c, trailer_tvb(offset + 4, mtd_length))
+	          metadata_tree:append_text(" JA4C: " .. trailer_tvb(offset + 4, mtd_length):string())
+	          metadata_tree:add(ndpi_fds.metadata_ja4c, trailer_tvb(offset + 4, mtd_length))
+	       elseif mtd_type == 3 then
+	          metadata_tree:append_text(" TLS Heuristic Fingerprint")
+	          tls_tree = metadata_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint, trailer_tvb(offset + 4, mtd_length))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_bytes0, trailer_tvb(offset + 4, 4))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_bytes1, trailer_tvb(offset + 8, 4))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_bytes2, trailer_tvb(offset + 12, 4))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_bytes3, trailer_tvb(offset + 16, 4))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_pkts0, trailer_tvb(offset + 20, 4))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_pkts1, trailer_tvb(offset + 24, 4))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_pkts2, trailer_tvb(offset + 28, 4))
+	          tls_tree:add(ndpi_fds.metadata_tls_heuristic_fingerprint_pkts3, trailer_tvb(offset + 32, 4))
 	       else
 		  -- Generic field
 		  metadata_tree:add(ndpi_fds.metadata_value, trailer_tvb(offset + 4, mtd_length))