Full network data removal from filtered dns.

extra/whitelist_domains.txt

@@ -10,6 +10,7 @@ client-office365-tas\.msedge\.net$
 \.skype\.com$
 api\.onedrive\.com$
 \.live\.com$
+\.bing.com$
 
 # Microsoft
 watson\.telemetry\.microsoft\.com$
@@ -20,11 +21,14 @@ adl\.windows\.com$
 fe2\.update\.microsoft\.com$
 fe3\.delivery\.mp\.microsoft\.com$
 download\.windowsupdate\.com$
-\.microsoft\.com
-\.windowsupdate\.com
+\.microsoft\.com$
+\.windowsupdate\.com$
 \.windows\.com$
 \.edgesuite\.net$
-blob\.core\.windows\.net$
+\.blob\.core\.windows\.net$
+
+\.windows\.com\.akadns\.net$
+\.microsoft\.com\.nsatc\.net$
 
 # Google
 update\.googleapis\.com$

modules/processing/network.py

@@ -180,16 +180,13 @@ def _add_hosts(self, connection):
         """Add IPs to unique list.
         @param connection: connection data
         """
-        if enabled_whitelist:
-            if connection["src"] in self.ip_whitelist:
-                 return False
-            if connection["dst"] in self.ip_whitelist:
-                 return False
         try:
             if connection["dst"] not in self.hosts:
                 ip = convert_to_printable(connection["dst"])
 
                 if ip not in self.hosts:
+                    if ip in self.ip_whitelist:
+                         return False
                     self.hosts.append(ip)
 
                     # We add external IPs to the list, only the first time
@@ -693,12 +690,6 @@ def run(self):
         self._process_smtp()
 
         # Remove hosts that have an IP which correlate to a whitelisted domain
-        if enabled_whitelist:
-            for delip in self.ip_whitelist:
-                if delip in self.unique_hosts:
-                    self.unique_hosts.remove(delip)
-                if delip in self.hosts:
-                    self.hosts.remove(delip)
 
         # Build results dict.
         self.results["hosts"] = self._enrich_hosts(self.unique_hosts)
@@ -711,6 +702,28 @@ def run(self):
         self.results["smtp"] = self.smtp_requests
         self.results["irc"] = self.irc_requests
 
+        if enabled_whitelist:
+
+            for host in self.results["hosts"]:
+                for delip in self.ip_whitelist:
+                    if delip == host["ip"]:
+                        self.results["hosts"].remove(host)
+
+            for host in self.results["tcp"]:
+                for delip in self.ip_whitelist:
+                    if delip == host["src"] or delip == host["dst"]:
+                        self.results["tcp"].remove(host)
+
+            for host in self.results["udp"]:
+                for delip in self.ip_whitelist:
+                    if delip == host["src"] or delip == host["dst"]:
+                        self.results["udp"].remove(host)
+
+            for host in self.results["icmp"]:
+                for delip in self.ip_whitelist:
+                    if delip == host["src"] or delip == host["dst"]:
+                        self.results["icmp"].remove(host)
+
         return self.results