Skip to content

Commit

Permalink
Add support for some Chinese shopping platforms (Temu, Shein and Taob…
Browse files Browse the repository at this point in the history 
…ao) (#2615)

Extend content match list
  • Loading branch information
@IvanNardi
IvanNardi authored Nov 12, 2024
1 parent 755bce2 commit 59ee1fe
Show file tree
Hide file tree
Showing 8 changed files with 94 additions and 10 deletions.
27 changes: 27 additions & 0 deletions doc/protocols.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1010,3 +1010,30 @@ References: `Main site: <https://www.paltalk.com/>`_
Naver is South Korea's largest search engine and online platform that offers various services including web search, email, news, shopping, cloud storage, maps, and social media features.

References: `Main site: <https://www.naver.com/>`_


.. _Proto 434:

`NDPI_PROTOCOL_SHEIN`
=====================
Shein is a fast fashion retailer.

References: `Main site <https://www.shein.com/>`_


.. _Proto 435:

`NDPI_PROTOCOL_TEMU`
====================
Temu is an online marketplace operated by the Chinese e-commerce company PDD Holdings.

References: `Main site <https://www.temu.com/>`_


.. _Proto 436:

`NDPI_PROTOCOL_TAOBAO`
======================
Taobao is a Chinese online shopping platform.

References: `Main site <https://www.taobao.com/>`_
3 changes: 3 additions & 0 deletions src/include/ndpi_protocol_ids.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,9 @@ typedef enum {
NDPI_PROTOCOL_DINGTALK = 431,
NDPI_PROTOCOL_PALTALK = 432,
NDPI_PROTOCOL_NAVER = 433,
NDPI_PROTOCOL_SHEIN = 434,
NDPI_PROTOCOL_TEMU = 435,
NDPI_PROTOCOL_TAOBAO = 436,

#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"
Expand Down
17 changes: 17 additions & 0 deletions src/lib/ndpi_content_match.c.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,7 @@ static ndpi_protocol_match host_match[] =
{ "location.live.net", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "virtualearth.net", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "trafficmanager.net", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "microsoftapp.net", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "testconnectivity.microsoft.com", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "teredo.ipv6.microsoft.com", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "teredo.ipv6.microsoft.com.nsatc.net", "Microsoft", NDPI_PROTOCOL_MICROSOFT, NDPI_PROTOCOL_CATEGORY_CONNECTIVITY_CHECK, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
Expand Down Expand Up @@ -1187,7 +1188,12 @@ static ndpi_protocol_match host_match[] =
{ "alicdn.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "aliyuncs.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "mmstat.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "alibabausercontent.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "alibabachengdun.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "alipayobjects.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "alipay.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "aliexpress.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "aliexpress-media.com", "Alibaba", NDPI_PROTOCOL_ALIBABA, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },

{ "mask.icloud.com", "iCloudPrivateRelay", NDPI_PROTOCOL_ICLOUD_PRIVATE_RELAY, NDPI_PROTOCOL_CATEGORY_VPN, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "mask-h2.icloud.com", "iCloudPrivateRelay", NDPI_PROTOCOL_ICLOUD_PRIVATE_RELAY, NDPI_PROTOCOL_CATEGORY_VPN, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
Expand Down Expand Up @@ -1528,6 +1534,8 @@ static ndpi_protocol_match host_match[] =
{ "mixpanel.com", "ADS_Analytic_Track", NDPI_PROTOCOL_ADS_ANALYTICS_TRACK, CUSTOM_CATEGORY_ADVERTISEMENT, NDPI_PROTOCOL_TRACKER_ADS, NDPI_PROTOCOL_DEFAULT_LEVEL },
/* Twitter ADS */
{ "ads-twitter.com", "ADS_Analytic_Track", NDPI_PROTOCOL_ADS_ANALYTICS_TRACK, CUSTOM_CATEGORY_ADVERTISEMENT, NDPI_PROTOCOL_TRACKER_ADS, NDPI_PROTOCOL_DEFAULT_LEVEL },
/* TANX (Taobao Ad Network and Exchange) is an advertising and marketing platform based in China */
{ "tanx.com", "ADS_Analytic_Track", NDPI_PROTOCOL_ADS_ANALYTICS_TRACK, CUSTOM_CATEGORY_ADVERTISEMENT, NDPI_PROTOCOL_TRACKER_ADS, NDPI_PROTOCOL_DEFAULT_LEVEL },

{ "xvideos.", "AdultContent", NDPI_PROTOCOL_ADULT_CONTENT, NDPI_PROTOCOL_CATEGORY_ADULT_CONTENT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "xvideos-games.com", "AdultContent", NDPI_PROTOCOL_ADULT_CONTENT, NDPI_PROTOCOL_CATEGORY_ADULT_CONTENT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
Expand Down Expand Up @@ -1696,6 +1704,15 @@ static ndpi_protocol_match host_match[] =
{ "navercorp.com", "Naver", NDPI_PROTOCOL_NAVER, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "ncloud.com", "Naver", NDPI_PROTOCOL_NAVER, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },

{ "shein.com", "Shein", NDPI_PROTOCOL_SHEIN, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "ltwebstatic.com", "Shein", NDPI_PROTOCOL_SHEIN, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },

{ "temu.com", "Temu", NDPI_PROTOCOL_TEMU, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "kwcdn.com", "Temu", NDPI_PROTOCOL_TEMU, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },

{ "taobao.com", "Taobao", NDPI_PROTOCOL_TAOBAO, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "tmall.com", "Taobao", NDPI_PROTOCOL_TAOBAO, NDPI_PROTOCOL_CATEGORY_SHOPPING, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },

#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_content_match_host_match.c.inc"
#endif
Expand Down
Binary file added tests/cfgs/default/pcap/sites2.pcapng
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ CustomProtocolC 3 222 1

Acceptable 8 592 3

1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.440/TLS.CustomProtocolA][IP: 440/CustomProtocolA][Encrypted][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][TCP Fingerprint: 2_64_64240_2e3cee914fc1/Linux][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.443/TLS.CustomProtocolA][IP: 443/CustomProtocolA][Encrypted][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][TCP Fingerprint: 2_64_64240_2e3cee914fc1/Linux][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 800/CustomProtocolC][IP: 800/CustomProtocolC][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][TCP Fingerprint: 2_64_64240_2e3cee914fc1/Linux][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 441/CustomProtocolB][IP: 441/CustomProtocolB][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][TCP Fingerprint: 2_64_64240_2e3cee914fc1/Linux][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 444/CustomProtocolB][IP: 444/CustomProtocolB][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][TCP Fingerprint: 2_64_64240_2e3cee914fc1/Linux][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
36 changes: 36 additions & 0 deletions tests/cfgs/default/result/sites2.pcapng.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
DPI Packets (TCP): 21 (7.00 pkts/flow)
Confidence DPI : 3 (flows)
Num dissector calls: 3 (1.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache fpc_dns: 0/3/0 (insert/search/found)
Automa host: 3/3 (search/found)
Automa domain: 3/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 0/0 (search/found)
Automa common alpns: 5/5 (search/found)
Patricia risk mask: 0/0 (search/found)
Patricia risk mask IPv6: 0/0 (search/found)
Patricia risk: 0/0 (search/found)
Patricia risk IPv6: 0/0 (search/found)
Patricia protocols: 4/2 (search/found)
Patricia protocols IPv6: 0/0 (search/found)

Shein 13 5080 1
Temu 20 7323 1
Taobao 15 7085 1

Acceptable 48 19488 3

JA3 Host Stats:
IP Address # JA3C
1 192.168.12.67 3


1 TCP 192.168.12.67:47694 <-> 20.15.0.9:443 [proto: 91.435/TLS.Temu][IP: 276/Azure][Encrypted][Confidence: DPI][FPC: 276/Azure, Confidence: IP address][DPI packets: 7][cat: Shopping/27][10 pkts/1963 bytes <-> 10 pkts/5360 bytes][Goodput ratio: 71/90][0.54 sec][Hostname/SNI: gtm.temu.com][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.464 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 61/49 282/342 86/112][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 196/536 571/1514 206/532][TCP Fingerprint: 2_64_65535_685ad951a756/Android][TLSv1.3][JA3C: 92768199641a57091d8ad9085387a16f][JA4: t13d1712h2_5b57614c22b0_3f5d972527c0][JA3S: 15af977ce25de452b96affa2addb1036][Safari][Cipher: TLS_AES_256_GCM_SHA384][Plen Bins: 8,8,8,0,0,0,0,0,0,25,0,0,0,0,0,0,16,8,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,16,0,0]
2 TCP 192.168.12.67:43446 <-> 59.82.122.224:443 [proto: 91.436/TLS.Taobao][IP: 274/Alibaba][Encrypted][Confidence: DPI][FPC: 274/Alibaba, Confidence: IP address][DPI packets: 8][cat: Shopping/27][9 pkts/2792 bytes <-> 6 pkts/4293 bytes][Goodput ratio: 82/92][0.78 sec][Hostname/SNI: umdc.taobao.com][(Advertised) ALPNs: http/1.1][(Negotiated) ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2;TLSv1.1;TLSv1][bytes ratio: -0.212 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 111/64 269/253 125/109][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 310/716 1078/1514 359/618][TCP Fingerprint: 2_64_65535_685ad951a756/Android][TLSv1.2][JA3C: 9b02ebd3a43b62d825e1ac605b621dc8][JA4: t13d1713ht_5b57614c22b0_eca864cca44a][ServerNames: *.alibabachengdun.com,*.alibabachengdun.net,umdc.aliapp.org,*.ynuf.aliapp.org,sgynuf.alibaba.com,pum.m.alibaba.com,ynuf.aliapp.org,mum.hzchengdun.com,mum.m.alibaba.com,umdc.alibaba-inc.com,umidiot.aliapp.org,us-mum.alibabachengdun.com,sg-pum.alibabachengdun.com,sg-pum.alibabachengdun.net,umdc.taobao.com,umdc.tmall.com,alibabachengdun.com][JA3S: 00447ab319e9d94ba2b4c1248e155917][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign Organization Validation CA - SHA256 - G3][Subject: C=CN, ST=ZheJiang, L=HangZhou, O=Alibaba (China) Technology Co., Ltd., CN=*.alibabachengdun.com][Certificate SHA-1: A4:84:85:BF:7A:3D:54:C0:EE:F2:8B:39:E7:ED:56:FB:74:6B:5E:61][Safari][Validity: 2024-09-11 08:46:01 - 2025-09-04 00:00:00][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 0,0,12,0,0,0,0,0,12,0,0,0,0,0,0,0,12,0,0,0,12,0,0,12,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,25,0,0]
3 TCP 192.168.12.67:46892 <-> 2.23.155.106:443 [proto: 91.434/TLS.Shein][IP: 0/Unknown][Encrypted][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Shopping/27][7 pkts/1067 bytes <-> 6 pkts/4013 bytes][Goodput ratio: 56/90][0.09 sec][Hostname/SNI: img.shein.com][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.580 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/5 58/19 21/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 152/669 583/1514 178/648][TCP Fingerprint: 2_64_65535_685ad951a756/Android][TLSv1.3][JA3C: f79b6bad2ad0641e1921aef10262856b][JA4: t13d1513h2_8daaf6152771_eca864cca44a][JA3S: 15af977ce25de452b96affa2addb1036][Safari][Cipher: TLS_AES_256_GCM_SHA384][Plen Bins: 0,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0,0,0,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,40,0,0]
Loading

0 comments on commit 59ee1fe

Please sign in to comment.