From 6890da44c947e59a7bd138c63bc529ebbb38fb15 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 2 Sep 2024 17:47:33 +0800 Subject: [PATCH 01/12] feat: implement no gob encoder. --- internal/msggateway/encoder.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/internal/msggateway/encoder.go b/internal/msggateway/encoder.go index 3af2663748..66d74ccb27 100644 --- a/internal/msggateway/encoder.go +++ b/internal/msggateway/encoder.go @@ -15,9 +15,6 @@ package msggateway import ( - "bytes" - "encoding/gob" - "github.com/openimsdk/tools/errs" ) @@ -33,19 +30,17 @@ func NewGobEncoder() *GobEncoder { } func (g *GobEncoder) Encode(data any) ([]byte, error) { - buff := bytes.Buffer{} - enc := gob.NewEncoder(&buff) - if err := enc.Encode(data); err != nil { - return nil, errs.WrapMsg(err, "GobEncoder.Encode failed", "action", "encode") + if b, ok := data.([]byte); ok { + return b, nil } - return buff.Bytes(), nil + return nil, errs.New("Encoder.Encode failed", "action", "encode") } func (g *GobEncoder) Decode(encodeData []byte, decodeData any) error { - buff := bytes.NewBuffer(encodeData) - dec := gob.NewDecoder(buff) - if err := dec.Decode(decodeData); err != nil { - return errs.WrapMsg(err, "GobEncoder.Decode failed", "action", "decode") + if b, ok := decodeData.(*[]byte); ok { + *b = encodeData + return nil } - return nil + + return errs.New("Encoder.Decode failed", "action", "decode") } From 165eecf0370886fb1403464bfadcec66884297d6 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Mon, 2 Sep 2024 18:05:49 +0800 Subject: [PATCH 02/12] update unitTest content. --- internal/msggateway/encoder_test.go | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 internal/msggateway/encoder_test.go diff --git a/internal/msggateway/encoder_test.go b/internal/msggateway/encoder_test.go new file mode 100644 index 0000000000..da77763ca9 --- /dev/null +++ b/internal/msggateway/encoder_test.go @@ -0,0 +1,48 @@ +package msggateway + +import ( + "testing" +) + +func TestGobEncoder_Encode(t *testing.T) { + encoder := NewGobEncoder() + + // 测试用例1: 编码 []byte 数据 + inputData := []byte("test data") + encodedData, err := encoder.Encode(inputData) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if string(encodedData) != string(inputData) { + t.Fatalf("expected encoded data to be '%s', got '%s'", inputData, encodedData) + } + + // 测试用例2: 编码非 []byte 数据 + nonByteData := "string data" + _, err = encoder.Encode(nonByteData) + if err == nil { + t.Fatalf("expected an error when encoding non-byte data, got none") + } +} + +func TestGobEncoder_Decode(t *testing.T) { + encoder := NewGobEncoder() + + // 测试用例1: 解码到 []byte 数据 + encodedData := []byte("test data") + var decodedData []byte + err := encoder.Decode(encodedData, &decodedData) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if string(decodedData) != string(encodedData) { + t.Fatalf("expected decoded data to be '%s', got '%s'", encodedData, decodedData) + } + + // 测试用例2: 解码到非 []byte 数据 + var nonByteData string + err = encoder.Decode(encodedData, &nonByteData) + if err == nil { + t.Fatalf("expected an error when decoding to non-byte data, got none") + } +} From 71f328ef94ab6372da8e22b8efb0026164aa6471 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:57:25 +0800 Subject: [PATCH 03/12] Update hub_server.go --- internal/msggateway/hub_server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 28c227162a..01482aa117 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -155,6 +155,7 @@ func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.M (client.IsBackground && client.PlatformID != constant.IOSPlatformID) { err := client.PushMessage(ctx, msgData) if err != nil { + log.ZWarn(ctx, "online push msg failed", err, "userID", userID, "platformID", client.PlatformID) userPlatform.ResultCode = int64(servererrs.ErrPushMsgErr.Code()) } else { if _, ok := s.pushTerminal[client.PlatformID]; ok { From 758606f62728211166e80f6e73784366ad85dffc Mon Sep 17 00:00:00 2001 From: Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Tue, 3 Sep 2024 18:20:52 +0800 Subject: [PATCH 04/12] fix: encoder replace to json encoder. --- internal/msggateway/encoder.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/msggateway/encoder.go b/internal/msggateway/encoder.go index 66d74ccb27..056f462367 100644 --- a/internal/msggateway/encoder.go +++ b/internal/msggateway/encoder.go @@ -15,6 +15,8 @@ package msggateway import ( + "encoding/json" + "github.com/openimsdk/tools/errs" ) @@ -30,17 +32,17 @@ func NewGobEncoder() *GobEncoder { } func (g *GobEncoder) Encode(data any) ([]byte, error) { - if b, ok := data.([]byte); ok { - return b, nil + b, err := json.Marshal(data) + if err != nil { + return nil, errs.New("Encoder.Encode failed", "action", "encode") } - return nil, errs.New("Encoder.Encode failed", "action", "encode") + return b, nil } func (g *GobEncoder) Decode(encodeData []byte, decodeData any) error { - if b, ok := decodeData.(*[]byte); ok { - *b = encodeData - return nil + err := json.Unmarshal(encodeData, decodeData) + if err != nil { + return errs.New("Encoder.Decode failed", "action", "decode") } - - return errs.New("Encoder.Decode failed", "action", "decode") + return nil } From 38a989b9fae30d6601835be68b84851d2608d3db Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:23:04 +0800 Subject: [PATCH 05/12] merge: merge main code into js branch. (#2648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: update group notification when set to null. (#2590) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * feat: update group notification when set to null. * update log standard. * feat: add long time push msg in prometheus (#2584) * feat: add long time push msg in prometheus * fix: log print * fix: go mod * fix: log msg * fix: log init * feat: push msg * feat: go mod ,remove cgo package * feat: remove error log * feat: test dummy push * feat:redis pool config * feat: push to kafka log * feat: supports getting messages based on session ID and seq (#2582) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: implement request batch count limit. (#2591) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: getting messages based on session ID and seq (#2595) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq --------- Co-authored-by: withchao * feat: avoid pulling messages from sessions with a large number of max seq values of 0 (#2602) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 --------- Co-authored-by: withchao * refactor: improve db structure in `storage/controller` (#2604) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * refactor: improve db structure in `storage/controller` * feat: implement offline push using kafka (#2600) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: implement offline push. * feat: implement batch Push spilt * update go mod * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: update Handler logic. * update MQ logic. * update * update * fix: update OfflinePushConsumerHandler. * feat: API supports gzip (#2609) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip --------- Co-authored-by: withchao * Fix err (#2608) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * feat: add rocksTimeout * feat: wrap logs * feat: add logs * feat: listen config * feat: enable listen TIME_WAIT port * feat: add logs * feat: cache batch * chore: enable fullUserCache * feat: push rpc num * feat: push err * feat: with operationID * feat: sleep * feat: change 1s * feat: change log * feat: implement Getbatch in rpcCache. * feat: print getOnline cost * feat: change log * feat: change kafka and push config * feat: del interface * feat: fix err * feat: change config * feat: go mod * feat: change config * feat: change config * feat: add sleep in push * feat: warn logs * feat: logs * feat: logs * feat: change port * feat: start config * feat: remove port reuse * feat: prometheus config * feat: prometheus config * feat: prometheus config * feat: add long time send msg to grafana * feat: init * feat: init * feat: implement offline push. * feat: batch get user online * feat: implement batch Push spilt * update go mod * Revert "feat: change port" This reverts commit 06d5e944 * feat: change port * feat: change config * feat: implement kafka producer and consumer. * update format, * add PushMQ log. * feat: get all online users and init push * feat: lock in online cache * feat: config * fix: init online status * fix: add logs * fix: userIDs * fix: add logs * feat: update Handler logic. * update MQ logic. * update * update * fix: method name * fix: update OfflinePushConsumerHandler. * fix: prommetrics * fix: add logs * fix: ctx * fix: log * fix: config * feat: change port * fix: atomic online cache status --------- Co-authored-by: Monet Lee * feature: add GetConversationsHasReadAndMaxSeq interface to the WebSocket API. (#2611) * fix: lru lock (#2613) * fix: lru lock * fix: lru lock * fix: lru lock * fix: nil pointer error on close (#2618) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close --------- Co-authored-by: withchao * feat: create group can push notification (#2617) * fix: blockage caused by listen error (#2620) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error --------- Co-authored-by: withchao * fix: go.mod (#2621) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod --------- Co-authored-by: withchao * feat: improve searchMsg implement. (#2614) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * remove unused script. * feat: improve searchMsg implement. * update mongo config. * Fix lock (#2622) * fix:log * fix: lock * fix: update setGroupInfoEX field name. (#2625) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name (#2626) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEX field name. * fix: update setGroupInfoEX field name * feat: msg gateway add log (#2631) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log --------- Co-authored-by: withchao * fix: update setGroupInfoEx func name and field. (#2634) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: update setGroupInfoEx func name and field. * refactor: update groupinfoEx field. * refactor: update database name in mongodb.yml * add groupName Condition * fix: fix setConversations req fill. (#2645) * refactor: refactor workflows contents. * add tool workflows. * update field. * fix: remove chat error. * Fix err. * fix error. * remove cn comment. * update workflows files. * update infra config. * move workflows. * feat: update bot. * fix: solve uncorrect outdated msg get. * update get docIDs logic. * update * update skip logic. * fix * update. * fix: delay deleteObject func. * remove unused content. * update log type. * feat: implement request batch count limit. * update * update * fix: fix setConversations req fill. * fix: GetMsgBySeqs boundary issues (#2647) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues --------- Co-authored-by: withchao * fix: the attribute version is obsolete, remove it (#2644) * refactor: update Userregister request field. (#2650) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: 蔡相跃 --- config/grafana-template/Demo.json | 3696 +++++++++-------- config/kafka.yml | 4 + config/openim-api.yml | 5 +- config/openim-msggateway.yml | 7 +- config/openim-msgtransfer.yml | 2 +- config/openim-push.yml | 13 +- config/openim-rpc-auth.yml | 5 +- config/openim-rpc-conversation.yml | 4 +- config/openim-rpc-friend.yml | 4 +- config/openim-rpc-group.yml | 6 +- config/openim-rpc-msg.yml | 7 +- config/openim-rpc-third.yml | 6 +- config/openim-rpc-user.yml | 8 +- config/prometheus.yml | 25 +- config/redis.yml | 1 + config/webhooks.yml | 4 +- docker-compose.yml | 3 - go.mod | 48 +- go.sum | 104 +- internal/api/group.go | 4 +- internal/api/router.go | 20 +- internal/msggateway/client.go | 7 +- internal/msggateway/constant.go | 2 + internal/msggateway/context.go | 6 +- internal/msggateway/init.go | 2 +- internal/msggateway/message_handler.go | 45 +- internal/msggateway/ws_server.go | 7 +- internal/msgtransfer/init.go | 22 +- .../msgtransfer/online_history_msg_handler.go | 46 +- .../online_msg_to_mongo_handler.go | 13 +- internal/push/offlinepush/dummy/push.go | 2 + internal/push/offlinepush/getui/push.go | 11 +- internal/push/offlinepush_handler.go | 122 + internal/push/onlinepusher.go | 4 +- internal/push/push.go | 19 +- internal/push/push_handler.go | 148 +- internal/rpc/conversation/conversaion.go | 4 +- internal/rpc/group/callback.go | 84 +- internal/rpc/group/db_map.go | 9 +- internal/rpc/group/group.go | 78 +- internal/rpc/group/notification.go | 43 +- internal/rpc/msg/clear.go | 2 +- internal/rpc/msg/sync_msg.go | 80 +- internal/rpc/relation/friend.go | 4 + internal/rpc/third/log.go | 3 +- internal/rpc/user/callback.go | 3 +- internal/rpc/user/online.go | 21 + internal/rpc/user/user.go | 8 +- internal/tools/cron_task.go | 14 +- pkg/callbackstruct/constant.go | 4 +- pkg/callbackstruct/group.go | 14 +- pkg/callbackstruct/user.go | 2 - pkg/common/cmd/push.go | 1 - pkg/common/cmd/root.go | 3 +- pkg/common/config/config.go | 41 +- pkg/common/prommetrics/grpc_push.go | 4 + pkg/common/prommetrics/rpc.go | 12 +- pkg/common/startrpc/start.go | 22 +- pkg/common/storage/cache/cachekey/online.go | 9 +- pkg/common/storage/cache/online.go | 1 + pkg/common/storage/cache/redis/batch.go | 6 +- .../storage/cache/redis/batch_handler.go | 8 +- pkg/common/storage/cache/redis/online.go | 32 + pkg/common/storage/cache/redis/token.go | 9 +- pkg/common/storage/controller/msg.go | 172 +- pkg/common/storage/controller/msg_transfer.go | 286 ++ pkg/common/storage/controller/push.go | 30 +- pkg/common/storage/controller/user.go | 7 +- pkg/localcache/lru/lru.go | 2 + pkg/localcache/lru/lru_expiration.go | 11 + pkg/localcache/lru/lru_lazy.go | 69 +- pkg/localcache/lru/lru_slot.go | 27 + pkg/rpccache/conversation.go | 2 +- pkg/rpccache/online.go | 268 +- pkg/rpcclient/msg.go | 21 +- pkg/rpcclient/user.go | 13 + pkg/util/conversationutil/conversationutil.go | 8 + scripts/create-topic.sh | 2 +- scripts/githooks/commit-msg.sh | 92 - scripts/githooks/pre-commit.sh | 111 - scripts/githooks/pre-push.sh | 119 - start-config.yml | 4 +- test/e2e/api/token/token.go | 6 +- tools/check-component/main.go | 13 +- 84 files changed, 3485 insertions(+), 2731 deletions(-) create mode 100644 internal/push/offlinepush_handler.go create mode 100644 pkg/common/storage/controller/msg_transfer.go delete mode 100644 scripts/githooks/commit-msg.sh delete mode 100644 scripts/githooks/pre-commit.sh delete mode 100644 scripts/githooks/pre-push.sh diff --git a/config/grafana-template/Demo.json b/config/grafana-template/Demo.json index c4668917f4..ea17d2c0ab 100644 --- a/config/grafana-template/Demo.json +++ b/config/grafana-template/Demo.json @@ -54,7 +54,7 @@ "liveNow": false, "panels": [ { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -62,1120 +62,1251 @@ "y": 0 }, "id": 35, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "panels": [], + "title": "Server", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Is the service up.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "description": "Is the service up.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "stepBefore", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 9, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bool_on_off" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 6, - "y": 1 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "insertNulls": false, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "lineWidth": 2, + "pointSize": 9, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "up", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "$legendName", - "range": true, - "refId": "A" - } - ], - "title": "UP", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_on_off" }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 6, + "y": 1 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of online users and login users within the time frame.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "online users" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#37bbff", - "mode": "fixed", - "seriesBy": "last" - } - } - ] - } - ] + "editorMode": "code", + "exemplar": false, + "expr": "up", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "$legendName", + "range": true, + "refId": "A" + } + ], + "title": "UP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of online users and login users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 12 - }, - "id": 37, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "online_user_num", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "online users", - "range": true, - "refId": "A" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(user_login_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "login num", - "range": true, - "refId": "B" + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Login Information", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of register users within the time frame.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "none" - }, - "overrides": [ { - "matcher": { - "id": "byName", - "options": "register users" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#7437ff", - "mode": "fixed", - "seriesBy": "last" - } - } - ] + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 59, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "online users" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#37bbff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "user_register_total", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "register users", - "range": true, - "refId": "A" - } - ], - "title": "Register num", - "type": "timeseries" + "editorMode": "code", + "exemplar": false, + "expr": "online_user_num", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "online users", + "range": true, + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of chat msg success.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "expr": "increase(user_login_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "login num", + "range": true, + "refId": "B" + } + ], + "title": "Login Information", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of register users within the time frame.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "unit": "none" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "register users" }, - "overrides": [] + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7437ff", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 23 + "editorMode": "code", + "exemplar": false, + "expr": "user_register_total", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "register users", + "range": true, + "refId": "A" + } + ], + "title": "Register num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of chat msg success.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 38, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(single_chat_msg_process_success_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "single msgs", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "increase(group_chat_msg_process_success_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "group msgs", - "range": true, - "refId": "B" - } - ], - "title": "Chat Msg Success Num", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of chat msg failed .", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(group_chat_msg_process_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Success Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of chat msg failed .", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "single msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#ff00dc", - "mode": "fixed", - "seriesBy": "last" - } - } - ] + "color": "green", + "value": null }, { - "matcher": { - "id": "byName", - "options": "group msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "#0cffef", - "mode": "fixed" - } - } - ] + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 39, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "single msgs" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#ff00dc", + "mode": "fixed", + "seriesBy": "last" + } + } + ] }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(single_chat_msg_process_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "single msgs", - "range": true, - "refId": "A" + { + "matcher": { + "id": "byName", + "options": "group msgs" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(group_chat_msg_process_failed_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "group msgs", - "range": true, - "refId": "B" - } - ], - "title": "Chat Msg Failed Num", - "type": "timeseries" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0cffef", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of msg failed offline pushed.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "failed msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed", - "seriesBy": "last" - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 6, - "x": 4, - "y": 33 - }, - "id": 42, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(msg_offline_push_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "failed msgs", - "range": true, - "refId": "A" - } - ], - "title": "Msg Offline Push Failed Num", - "type": "timeseries" + "editorMode": "code", + "exemplar": false, + "expr": "increase(single_chat_msg_process_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "single msgs", + "range": true, + "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of failed set seq.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + "editorMode": "code", + "expr": "increase(group_chat_msg_process_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "group msgs", + "range": true, + "refId": "B" + } + ], + "title": "Chat Msg Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of msg failed offline pushed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [ + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "failed msgs" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed", - "seriesBy": "last" - } - } - ] + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "gridPos": { - "h": 11, - "w": 6, - "x": 14, - "y": 33 - }, - "id": 43, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(seq_set_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "failed addr: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Seq Set Failed Num", - "type": "timeseries" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 42, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of successfully inserted messages.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_offline_push_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr:{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Msg Offline Push Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of failed set seq.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed", + "seriesBy": "last" } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(seq_set_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Seq Set Failed Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of messages that take a long time to send.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "failed msgs" }, - "overrides": [] + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed", + "seriesBy": "last" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 44 + "editorMode": "code", + "exemplar": false, + "expr": "msg_long_time_push_total", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "addr:{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Long Time Send Msg Total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of successfully inserted messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 44, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(msg_insert_redis_success_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "redis: {{instance}}", - "range": true, - "refId": "A" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_success_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(msg_insert_mongo_success_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Msg Success Insert Num", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of failed insertion messages.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(msg_insert_mongo_success_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "mongo: {{instance}}", - "range": true, - "refId": "B" + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Msg Success Insert Num", - "type": "timeseries" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "This metric represents the number of failed insertion messages.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 45, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "editorMode": "code", + "exemplar": false, + "expr": "increase(msg_insert_redis_failed_total[$time])", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "redis: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "increase(msg_insert_redis_failed_total[$time])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "redis: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "increase(msg_insert_mongo_failed_total[$time])", - "hide": false, - "instant": false, - "legendFormat": "mongo: {{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "Msg Failed Insert Num", - "type": "timeseries" + "editorMode": "code", + "expr": "increase(msg_insert_mongo_failed_total[$time])", + "hide": false, + "instant": false, + "legendFormat": "mongo: {{instance}}", + "range": true, + "refId": "B" } ], - "title": "Server", - "type": "row" + "title": "Msg Failed Insert Num", + "type": "timeseries" }, { "collapsed": true, @@ -1183,7 +1314,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 54 }, "id": 22, "panels": [ @@ -1973,7 +2104,7 @@ "h": 1, "w": 24, "x": 0, - "y": 2 + "y": 55 }, "id": 28, "panels": [ @@ -2827,7 +2958,7 @@ "h": 1, "w": 24, "x": 0, - "y": 3 + "y": 56 }, "id": 25, "panels": [ @@ -3377,849 +3508,848 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 57 }, "id": 6, - "panels": [], - "title": "Process", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Usage Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 5 }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job=~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "CPU Usage Percentage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of open file descriptors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the proportion of CPU runtime within 1 second. It is calculated as the average CPU runtime over 1 minute.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n rate(process_cpu_seconds_total{job!~\"$rpcNameFilter\"}[1m])*100,\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage Percentage", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Open File Descriptors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of open file descriptors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of open file descriptors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Open File Descriptors", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_open_fds{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Open File Descriptors", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process virtual memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "This metric represents the number of process virtual memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 27 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "unit": "bytes" + }, + "overrides": [] }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_virtual_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Virtual Memory bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process virtual memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 27 }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 27 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Virtual Memory bytes", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_virtual_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Virtual Memory bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process resident memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_resident_memory_bytes{job=~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Resident Memory bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "This metric represents the number of process resident memory bytes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "description": "This metric represents the number of process resident memory bytes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "10.3.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.3.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{job}}: {{instance}}", - "range": true, - "refId": "A" + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\r\n process_resident_memory_bytes{job!~\"$rpcNameFilter\"},\r\n \"job\",\r\n \"$1\",\r\n \"job\",\r\n \".*openim-(.*)\"\r\n)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{job}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory bytes", + "type": "timeseries" } ], - "title": "Resident Memory bytes", - "type": "timeseries" + "title": "Process", + "type": "row" }, { "collapsed": true, @@ -4227,7 +4357,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 58 }, "id": 3, "panels": [ @@ -5441,6 +5571,6 @@ "timezone": "", "title": "Demo", "uid": "a506d250-b606-4702-86a7-ac6aa1d069a1", - "version": 23, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/config/kafka.yml b/config/kafka.yml index e45474f275..fd06ae2bb4 100644 --- a/config/kafka.yml +++ b/config/kafka.yml @@ -14,12 +14,16 @@ toRedisTopic: toRedis toMongoTopic: toMongo # Kafka topic for push notifications toPushTopic: toPush +# Kafka topic for offline push notifications +toOfflinePushTopic: toOfflinePush # Consumer group ID for Redis topic toRedisGroupID: redis # Consumer group ID for MongoDB topic toMongoGroupID: mongo # Consumer group ID for push notifications topic toPushGroupID: push +# Consumer group ID for offline push notifications topic +toOfflinePushGroupID: offlinePush # TLS (Transport Layer Security) configuration tls: # Enable or disable TLS diff --git a/config/openim-api.yml b/config/openim-api.yml index 9f53038d88..4c38e1005b 100644 --- a/config/openim-api.yml +++ b/config/openim-api.yml @@ -3,11 +3,14 @@ api: listenIP: 0.0.0.0 # Listening ports; if multiple are configured, multiple instances will be launched, must be consistent with the number of prometheus.ports ports: [ 10002 ] + # API compression level; 0: default compression, 1: best compression, 2: best speed, -1: no compression + compressionLevel: 0 + prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must match the number of api.ports - ports: [ 20502 ] + ports: [ 12002 ] # This address can be accessed via a browser grafanaURL: http://127.0.0.1:13000/ diff --git a/config/openim-msggateway.yml b/config/openim-msggateway.yml index 63332740ff..428f3ba476 100644 --- a/config/openim-msggateway.yml +++ b/config/openim-msggateway.yml @@ -2,13 +2,13 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP registerIP: # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10140 ] + ports: [ 10140, 10141, 10142, 10143, 10144, 10145, 10146, 10147, 10148, 10149, 10150, 10151, 10152, 10153, 10154, 10155 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20640 ] + ports: [ 12140, 12141, 12142, 12143, 12144, 12145, 12146, 12147, 12148, 12149, 12150, 12151, 12152, 12153, 12154, 12155 ] # IP address that the RPC/WebSocket service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 @@ -25,6 +25,3 @@ longConnSvr: # 1: For Android, iOS, Windows, Mac, and web platforms, only one instance can be online at a time multiLoginPolicy: 1 - - - diff --git a/config/openim-msgtransfer.yml b/config/openim-msgtransfer.yml index e71a218edf..753ac10bc0 100644 --- a/config/openim-msgtransfer.yml +++ b/config/openim-msgtransfer.yml @@ -3,4 +3,4 @@ prometheus: enable: true # List of ports that Prometheus listens on; each port corresponds to an instance of monitoring. Ensure these are managed accordingly # Because four instances have been launched, four ports need to be specified - ports: [ 20600, 20601, 20602, 20603 ] + ports: [ 12020, 12021, 12022, 12023, 12024, 12025, 12026, 12027 ] diff --git a/config/openim-push.yml b/config/openim-push.yml index 70aa5997f5..6df2a62b7f 100644 --- a/config/openim-push.yml +++ b/config/openim-push.yml @@ -1,16 +1,16 @@ rpc: # The IP address where this RPC service registers itself; if left blank, it defaults to the internal network IP - registerIP: + registerIP: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10170, 10171, 10172, 10173 ] + ports: [ 10170, 10171, 10172, 10173, 10174, 10175, 10176, 10177, 10178, 10179, 10180, 10181, 10182, 10183, 10184, 10185 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20670, 20671, 20672, 20673 ] + ports: [ 12170, 12171, 12172, 12173, 12174, 12175, 12176, 12177, 12178, 12179, 12180, 12181, 12182, 12183, 12184, 12185 ] maxConcurrentWorkers: 3 #Use geTui for offline push notifications, or choose fcm or jpns; corresponding configuration settings must be specified. @@ -38,9 +38,4 @@ iosPush: badgeCount: true production: false - - - - - - +fullUserCache: true diff --git a/config/openim-rpc-auth.yml b/config/openim-rpc-auth.yml index c55c745b6c..496803e43b 100644 --- a/config/openim-rpc-auth.yml +++ b/config/openim-rpc-auth.yml @@ -4,15 +4,14 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10160 ] + ports: [ 10200 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20660 ] + ports: [ 12200 ] tokenPolicy: # Token validity period, in days expire: 90 - diff --git a/config/openim-rpc-conversation.yml b/config/openim-rpc-conversation.yml index 00c9c5aab9..3581d7e19e 100644 --- a/config/openim-rpc-conversation.yml +++ b/config/openim-rpc-conversation.yml @@ -4,10 +4,10 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10180 ] + ports: [ 10220 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20680 ] + ports: [ 12220 ] diff --git a/config/openim-rpc-friend.yml b/config/openim-rpc-friend.yml index afac3c5db9..3022c09f32 100644 --- a/config/openim-rpc-friend.yml +++ b/config/openim-rpc-friend.yml @@ -4,10 +4,10 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10120 ] + ports: [ 10240 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20620 ] + ports: [ 12240 ] diff --git a/config/openim-rpc-group.yml b/config/openim-rpc-group.yml index d0243b5fbb..9a634d12ff 100644 --- a/config/openim-rpc-group.yml +++ b/config/openim-rpc-group.yml @@ -4,13 +4,13 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10150 ] + ports: [ 10260 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20650 ] + ports: [ 12260 ] -enableHistoryForNewMembers: true \ No newline at end of file +enableHistoryForNewMembers: true diff --git a/config/openim-rpc-msg.yml b/config/openim-rpc-msg.yml index 15840c7f3b..82d6e2f539 100644 --- a/config/openim-rpc-msg.yml +++ b/config/openim-rpc-msg.yml @@ -4,17 +4,14 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10130 ] + ports: [ 10280 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20630 ] + ports: [ 12280 ] # Does sending messages require friend verification friendVerify: false - - - diff --git a/config/openim-rpc-third.yml b/config/openim-rpc-third.yml index 512ca391fa..d8f2d427f2 100644 --- a/config/openim-rpc-third.yml +++ b/config/openim-rpc-third.yml @@ -4,13 +4,13 @@ rpc: # IP address that the RPC service listens on; setting to 0.0.0.0 listens on both internal and external IPs. If left blank, it automatically uses the internal network IP listenIP: 0.0.0.0 # List of ports that the RPC service listens on; configuring multiple ports will launch multiple instances. These must match the number of configured prometheus ports - ports: [ 10190 ] + ports: [ 10300 ] prometheus: # Enable or disable Prometheus monitoring enable: true # List of ports that Prometheus listens on; these must match the number of rpc.ports to ensure correct monitoring setup - ports: [ 20690 ] + ports: [ 12300 ] object: @@ -37,4 +37,4 @@ object: accessKeyID: accessKeySecret: sessionToken: - publicRead: false \ No newline at end of file + publicRead: false diff --git a/config/openim-rpc-user.yml b/config/openim-rpc-user.yml index 1958bbc6ab..798105472c 100644 --- a/config/openim-rpc-user.yml +++ b/config/openim-rpc-user.yml @@ -4,14 +4,10 @@ rpc: # Listening IP; 0.0.0.0 means both internal and external IPs are listened to, if blank, the internal network IP is automatically obtained by default listenIP: 0.0.0.0 # Listening ports; if multiple are configured, multiple instances will be launched, and must be consistent with the number of prometheus.ports - ports: [ 10110 ] + ports: [ 10320 ] prometheus: # Whether to enable prometheus enable: true # Prometheus listening ports, must be consistent with the number of rpc.ports - ports: [ 20610 ] - - - - + ports: [ 12320 ] diff --git a/config/prometheus.yml b/config/prometheus.yml index 627cf94114..4f0f7e32c0 100644 --- a/config/prometheus.yml +++ b/config/prometheus.yml @@ -28,56 +28,59 @@ scrape_configs: - targets: [ internal_ip:20500 ] - job_name: openimserver-openim-api static_configs: - - targets: [ internal_ip:20502 ] + - targets: [ internal_ip:12002 ] labels: namespace: default - job_name: openimserver-openim-msggateway static_configs: - - targets: [ internal_ip:20640 ] + - targets: [ internal_ip:12140 ] +# - targets: [ internal_ip:12140, internal_ip:12141, internal_ip:12142, internal_ip:12143, internal_ip:12144, internal_ip:12145, internal_ip:12146, internal_ip:12147, internal_ip:12148, internal_ip:12149, internal_ip:12150, internal_ip:12151, internal_ip:12152, internal_ip:12153, internal_ip:12154, internal_ip:12155 ] labels: namespace: default - job_name: openimserver-openim-msgtransfer static_configs: - - targets: [ internal_ip:20600, internal_ip:20601, internal_ip:20602, internal_ip:20603 ] + - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027 ] +# - targets: [ internal_ip:12020, internal_ip:12021, internal_ip:12022, internal_ip:12023, internal_ip:12024, internal_ip:12025, internal_ip:12026, internal_ip:12027, internal_ip:12028, internal_ip:12029, internal_ip:12030, internal_ip:12031, internal_ip:12032, internal_ip:12033, internal_ip:12034, internal_ip:12035 ] labels: namespace: default - job_name: openimserver-openim-push static_configs: - - targets: [ internal_ip:20670, internal_ip:20671, internal_ip:20672, internal_ip:20673] + - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177 ] +# - targets: [ internal_ip:12170, internal_ip:12171, internal_ip:12172, internal_ip:12173, internal_ip:12174, internal_ip:12175, internal_ip:12176, internal_ip:12177, internal_ip:12178, internal_ip:12179, internal_ip:12180, internal_ip:12181, internal_ip:12182, internal_ip:12183, internal_ip:12184, internal_ip:12185 ] labels: namespace: default - job_name: openimserver-openim-rpc-auth static_configs: - - targets: [ internal_ip:20600 ] + - targets: [ internal_ip:12200 ] labels: namespace: default - job_name: openimserver-openim-rpc-conversation static_configs: - - targets: [ internal_ip:20680 ] + - targets: [ internal_ip:12220 ] labels: namespace: default - job_name: openimserver-openim-rpc-friend static_configs: - - targets: [ internal_ip:20620 ] + - targets: [ internal_ip:12240 ] labels: namespace: default - job_name: openimserver-openim-rpc-group static_configs: - - targets: [ internal_ip:20650 ] + - targets: [ internal_ip:12260 ] labels: namespace: default - job_name: openimserver-openim-rpc-msg static_configs: - - targets: [ internal_ip:20630 ] + - targets: [ internal_ip:12280 ] labels: namespace: default - job_name: openimserver-openim-rpc-third static_configs: - - targets: [ internal_ip:20690 ] + - targets: [ internal_ip:12300 ] labels: namespace: default - job_name: openimserver-openim-rpc-user static_configs: - - targets: [ internal_ip:20610 ] + - targets: [ internal_ip:12320 ] labels: namespace: default \ No newline at end of file diff --git a/config/redis.yml b/config/redis.yml index 83e3054590..2448bcb5c6 100644 --- a/config/redis.yml +++ b/config/redis.yml @@ -4,3 +4,4 @@ password: openIM123 clusterMode: false db: 0 maxRetry: 10 +poolSize: 100 diff --git a/config/webhooks.yml b/config/webhooks.yml index 24fb2413a0..eee79c650e 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -130,10 +130,10 @@ beforeSetGroupInfo: enable: false timeout: 5 failedContinue: true -afterSetGroupInfoEX: +afterSetGroupInfoEx: enable: false timeout: 5 -beforeSetGroupInfoEX: +beforeSetGroupInfoEx: enable: false timeout: 5 failedContinue: true diff --git a/docker-compose.yml b/docker-compose.yml index 49c44d3d9a..edac65b13e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3' - networks: openim: driver: bridge @@ -186,4 +184,3 @@ services: # networks: # - openim - diff --git a/go.mod b/go.mod index ae30db0565..5fab509914 100644 --- a/go.mod +++ b/go.mod @@ -6,21 +6,21 @@ require ( firebase.google.com/go v3.13.0+incompatible github.com/dtm-labs/rockscache v0.1.1 github.com/gin-gonic/gin v1.9.1 - github.com/go-playground/validator/v10 v10.18.0 + github.com/go-playground/validator/v10 v10.20.0 github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.9 - github.com/openimsdk/tools v0.0.49-alpha.55 + github.com/openimsdk/protocol v0.0.72-alpha.25 + github.com/openimsdk/tools v0.0.50-alpha.12 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.14.0 google.golang.org/api v0.165.0 - google.golang.org/grpc v1.62.1 - google.golang.org/protobuf v1.33.0 + google.golang.org/grpc v1.66.2 + google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v3 v3.0.1 ) @@ -29,6 +29,7 @@ require github.com/google/uuid v1.6.0 require ( github.com/IBM/sarama v1.43.0 github.com/fatih/color v1.14.1 + github.com/gin-contrib/gzip v1.0.1 github.com/go-redis/redis v6.15.9+incompatible github.com/go-redis/redismock/v9 v9.2.0 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -42,13 +43,12 @@ require ( github.com/stathat/consistent v1.0.0 go.uber.org/automaxprocs v1.5.3 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.8.0 ) require ( cloud.google.com/go v0.112.0 // indirect - cloud.google.com/go/compute v1.23.3 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/firestore v1.14.0 // indirect cloud.google.com/go/iam v1.1.5 // indirect cloud.google.com/go/longrunning v0.5.4 // indirect @@ -73,10 +73,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.25.4 // indirect github.com/aws/smithy-go v1.17.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -117,7 +119,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kelindar/simd v1.1.2 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/lithammer/shortuuid v3.0.0+incompatible // indirect @@ -132,7 +134,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect github.com/mozillazg/go-httpheader v0.4.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -169,17 +171,17 @@ require ( go.opentelemetry.io/otel/trace v1.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.3.0 // indirect + golang.org/x/arch v0.7.0 // indirect golang.org/x/image v0.15.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/oauth2 v0.17.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gorm.io/gorm v1.25.8 // indirect stathat.com/c/consistent v1.0.0 // indirect ) @@ -187,10 +189,10 @@ require ( require ( github.com/go-playground/locales v0.14.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/spf13/cobra v1.8.0 - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.27.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 815b6badfc..4c3b4aa21c 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/firestore v1.14.0 h1:8aLcKnMPoldYU3YHgu4t2exrKhLQkqaXAGqT0ljrFVw= cloud.google.com/go/firestore v1.14.0/go.mod h1:96MVaHLsEhbvkBEdZgfN+AS/GIkco1LRpH9Xp9YZfzQ= cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= @@ -65,21 +63,23 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= -github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= @@ -121,6 +121,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/gzip v1.0.1 h1:HQ8ENHODeLY7a4g1Au/46Z92bdGFl74OhxcZble9WJE= +github.com/gin-contrib/gzip v1.0.1/go.mod h1:njt428fdUNRvjuJf16tZMYZ2Yl+WQB53X5wmhDwXvC4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= @@ -144,8 +146,8 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= -github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U= -github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= @@ -257,8 +259,9 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -288,8 +291,8 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -319,12 +322,12 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.9 h1:Dyx4vs88IU4rJ2YcP/TdYp4ww8JjsMkV89hB/Eazx+A= -github.com/openimsdk/protocol v0.0.72-alpha.9/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k= -github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/openimsdk/protocol v0.0.72-alpha.25 h1:W8E6gnwt5V6anr/8lYOf5v/Lcsggf7gIAzJbw7YU6So= +github.com/openimsdk/protocol v0.0.72-alpha.25/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.50-alpha.12 h1:rV3BxgqN+F79vZvdoQ+97Eob8ScsRVEM8D+Wrcl23uo= +github.com/openimsdk/tools v0.0.50-alpha.12/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -408,8 +411,8 @@ github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -456,8 +459,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -465,8 +468,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= @@ -493,19 +496,19 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= -golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -520,8 +523,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -534,8 +537,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -564,17 +567,17 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe h1:USL2DhxfgRchafRvt/wYyyQNzwgL7ZiURcozOE/Pkvo= google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= -google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -586,8 +589,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -608,6 +611,7 @@ gorm.io/gorm v1.25.8 h1:WAGEZ/aEcznN4D03laj8DKnehe1e9gYQAjW8xyPRdeo= gorm.io/gorm v1.25.8/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/internal/api/group.go b/internal/api/group.go index 14f50cacdd..3af39ae455 100644 --- a/internal/api/group.go +++ b/internal/api/group.go @@ -35,8 +35,8 @@ func (o *GroupApi) SetGroupInfo(c *gin.Context) { a2r.Call(group.GroupClient.SetGroupInfo, o.Client, c) } -func (o *GroupApi) SetGroupInfoEX(c *gin.Context) { - a2r.Call(group.GroupClient.SetGroupInfoEX, o.Client, c) +func (o *GroupApi) SetGroupInfoEx(c *gin.Context) { + a2r.Call(group.GroupClient.SetGroupInfoEx, o.Client, c) } func (o *GroupApi) JoinGroup(c *gin.Context) { diff --git a/internal/api/router.go b/internal/api/router.go index 1718505817..37d0347ad2 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "github.com/gin-contrib/gzip" "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" @@ -22,6 +23,13 @@ import ( "github.com/openimsdk/tools/mw" ) +const ( + NoCompression = -1 + DefaultCompression = 0 + BestCompression = 1 + BestSpeed = 2 +) + func prommetricsGin() gin.HandlerFunc { return func(c *gin.Context) { c.Next() @@ -54,7 +62,15 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En conversationRpc := rpcclient.NewConversation(disCov, config.Share.RpcRegisterName.Conversation) authRpc := rpcclient.NewAuth(disCov, config.Share.RpcRegisterName.Auth) thirdRpc := rpcclient.NewThird(disCov, config.Share.RpcRegisterName.Third, config.API.Prometheus.GrafanaURL) - + switch config.API.Api.CompressionLevel { + case NoCompression: + case DefaultCompression: + r.Use(gzip.Gzip(gzip.DefaultCompression)) + case BestCompression: + r.Use(gzip.Gzip(gzip.BestCompression)) + case BestSpeed: + r.Use(gzip.Gzip(gzip.BestSpeed)) + } r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) @@ -114,7 +130,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En { groupRouterGroup.POST("/create_group", g.CreateGroup) groupRouterGroup.POST("/set_group_info", g.SetGroupInfo) - groupRouterGroup.POST("/set_group_info_ex", g.SetGroupInfoEX) + groupRouterGroup.POST("/set_group_info_ex", g.SetGroupInfoEx) groupRouterGroup.POST("/join_group", g.JoinGroup) groupRouterGroup.POST("/quit_group", g.QuitGroup) groupRouterGroup.POST("/group_application_response", g.ApplicationGroupResponse) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index dcb15b70da..bc06fa9507 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -22,6 +22,8 @@ import ( "sync/atomic" "time" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" @@ -30,7 +32,6 @@ import ( "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/stringutil" - "google.golang.org/protobuf/proto" ) var ( @@ -220,6 +221,10 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.SendSignalMessage(ctx, binaryReq) case WSPullMsgBySeqList: resp, messageErr = c.longConnServer.PullMessageBySeqList(ctx, binaryReq) + case WSPullMsg: + resp, messageErr = c.longConnServer.GetSeqMessage(ctx, binaryReq) + case WSGetConvMaxReadSeq: + resp, messageErr = c.longConnServer.GetConversationsHasReadAndMaxSeq(ctx, binaryReq) case WsLogoutMsg: resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index dc5ad77861..584cebe1e1 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -39,6 +39,8 @@ const ( WSPullMsgBySeqList = 1002 WSSendMsg = 1003 WSSendSignalMsg = 1004 + WSPullMsg = 1005 + WSGetConvMaxReadSeq = 1006 WSPushMsg = 2001 WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 diff --git a/internal/msggateway/context.go b/internal/msggateway/context.go index 6c80ece1ba..3909766b1b 100644 --- a/internal/msggateway/context.go +++ b/internal/msggateway/context.go @@ -66,12 +66,16 @@ func (c *UserConnContext) Value(key any) any { } func newContext(respWriter http.ResponseWriter, req *http.Request) *UserConnContext { + remoteAddr := req.RemoteAddr + if forwarded := req.Header.Get("X-Forwarded-For"); forwarded != "" { + remoteAddr += "_" + forwarded + } return &UserConnContext{ RespWriter: respWriter, Req: req, Path: req.URL.Path, Method: req.Method, - RemoteAddr: req.RemoteAddr, + RemoteAddr: remoteAddr, ConnID: encrypt.Md5(req.RemoteAddr + "_" + strconv.Itoa(int(timeutil.GetCurrentTimestampByMill()))), } } diff --git a/internal/msggateway/init.go b/internal/msggateway/init.go index 44e79e4122..50da060976 100644 --- a/internal/msggateway/init.go +++ b/internal/msggateway/init.go @@ -58,7 +58,7 @@ func Start(ctx context.Context, index int, conf *Config) error { ) hubServer := NewServer(rpcPort, longServer, conf, func(srv *Server) error { - longServer.online = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, longServer.subscriberUserOnlineStatusChanges) + longServer.online, _ = rpccache.NewOnlineCache(srv.userRcp, nil, rdb, false, longServer.subscriberUserOnlineStatusChanges) return nil }) diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 8a11e6ab3c..4b78c10048 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -19,6 +19,8 @@ import ( "sync" "github.com/go-playground/validator/v10" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/protocol/msg" @@ -27,7 +29,6 @@ import ( "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/utils/jsonutil" - "google.golang.org/protobuf/proto" ) type Req struct { @@ -94,6 +95,8 @@ type MessageHandler interface { SendMessage(context context.Context, data *Req) ([]byte, error) SendSignalMessage(context context.Context, data *Req) ([]byte, error) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) + GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) + GetSeqMessage(context context.Context, data *Req) ([]byte, error) UserLogout(context context.Context, data *Req) ([]byte, error) SetUserDeviceBackground(context context.Context, data *Req) ([]byte, bool, error) } @@ -175,7 +178,7 @@ func (g GrpcHandler) SendSignalMessage(context context.Context, data *Req) ([]by func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([]byte, error) { req := sdkws.PullMessageBySeqsReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { - return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") + return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "PullMessageBySeqsReq") } if err := g.validate.Struct(data); err != nil { return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "PullMessageBySeqsReq") @@ -191,6 +194,44 @@ func (g GrpcHandler) PullMessageBySeqList(context context.Context, data *Req) ([ return c, nil } +func (g GrpcHandler) GetConversationsHasReadAndMaxSeq(context context.Context, data *Req) ([]byte, error) { + req := msg.GetConversationsHasReadAndMaxSeqReq{} + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, errs.WrapMsg(err, "err proto unmarshal", "action", "unmarshal", "dataType", "GetConversationsHasReadAndMaxSeq") + } + if err := g.validate.Struct(data); err != nil { + return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetConversationsHasReadAndMaxSeq") + } + resp, err := g.msgRpcClient.GetConversationsHasReadAndMaxSeq(context, &req) + if err != nil { + return nil, err + } + c, err := proto.Marshal(resp) + if err != nil { + return nil, errs.WrapMsg(err, "error marshaling response", "action", "marshal", "dataType", "GetConversationsHasReadAndMaxSeq") + } + return c, nil +} + +func (g GrpcHandler) GetSeqMessage(context context.Context, data *Req) ([]byte, error) { + req := msg.GetSeqMessageReq{} + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, errs.WrapMsg(err, "error unmarshaling request", "action", "unmarshal", "dataType", "GetSeqMessage") + } + if err := g.validate.Struct(data); err != nil { + return nil, errs.WrapMsg(err, "validation failed", "action", "validate", "dataType", "GetSeqMessage") + } + resp, err := g.msgRpcClient.GetSeqMessage(context, &req) + if err != nil { + return nil, err + } + c, err := proto.Marshal(resp) + if err != nil { + return nil, errs.WrapMsg(err, "error marshaling response", "action", "marshal", "dataType", "GetSeqMessage") + } + return c, nil +} + func (g GrpcHandler) UserLogout(context context.Context, data *Req) ([]byte, error) { req := push.DelUserPushTokenReq{} if err := proto.Unmarshal(data.Data, &req); err != nil { diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index 9b18ade7dc..60e2b8d537 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -265,7 +265,7 @@ func (ws *WsServer) registerClient(client *Client) { if clientOK { ws.clients.Set(client.UserID, client) // There is already a connection to the platform - log.ZInfo(client.ctx, "repeat login", "userID", client.UserID, "platformID", + log.ZDebug(client.ctx, "repeat login", "userID", client.UserID, "platformID", client.PlatformID, "old remote addr", getRemoteAdders(oldClients)) ws.onlineUserConnNum.Add(1) } else { @@ -293,7 +293,7 @@ func (ws *WsServer) registerClient(client *Client) { wg.Wait() - log.ZInfo( + log.ZDebug( client.ctx, "user online", "online user Num", @@ -360,7 +360,7 @@ func (ws *WsServer) unregisterClient(client *Client) { ws.onlineUserConnNum.Add(-1) ws.subscription.DelClient(client) //ws.SetUserOnlineStatus(client.ctx, client, constant.Offline) - log.ZInfo(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", + log.ZDebug(client.ctx, "user offline", "close reason", client.closedErr, "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load(), ) @@ -425,6 +425,7 @@ func (ws *WsServer) wsHandler(w http.ResponseWriter, r *http.Request) { return } + log.ZDebug(connContext, "new conn", "token", connContext.GetToken()) // Create a WebSocket long connection object wsLongConn := newGWebSocket(WebSocket, ws.handshakeTimeout, ws.writeBufferSize) if err := wsLongConn.GenerateLongConn(w, r); err != nil { diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 5e0ccd0e52..7dc2ebeea0 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -18,19 +18,20 @@ import ( "context" "errors" "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" "github.com/openimsdk/tools/db/mongoutil" "github.com/openimsdk/tools/db/redisutil" "github.com/openimsdk/tools/utils/datautil" - "net/http" - "os" - "os/signal" - "syscall" "github.com/openimsdk/open-im-server/v3/pkg/common/config" - kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + discRegister "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/tools/errs" @@ -65,6 +66,7 @@ type Config struct { func Start(ctx context.Context, index int, config *Config) error { log.CInfo(ctx, "MSG-TRANSFER server is initializing", "prometheusPorts", config.MsgTransfer.Prometheus.Ports, "index", index) + mgocli, err := mongoutil.NewMongoDB(ctx, config.MongodbConfig.Build()) if err != nil { return err @@ -73,12 +75,13 @@ func Start(ctx context.Context, index int, config *Config) error { if err != nil { return err } - client, err := kdisc.NewDiscoveryRegister(&config.Discovery, &config.Share) + client, err := discRegister.NewDiscoveryRegister(&config.Discovery, &config.Share) if err != nil { return err } client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"LoadBalancingPolicy": "%s"}`, "round_robin"))) + msgModel := redis.NewMsgCache(rdb) msgDocModel, err := mgo.NewMsgMongo(mgocli.GetDB()) if err != nil { @@ -94,20 +97,21 @@ func Start(ctx context.Context, index int, config *Config) error { return err } seqUserCache := redis.NewSeqUserCacheRedis(rdb, seqUser) - msgDatabase, err := controller.NewCommonMsgDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) + msgTransferDatabase, err := controller.NewMsgTransferDatabase(msgDocModel, msgModel, seqUserCache, seqConversationCache, &config.KafkaConfig) if err != nil { return err } conversationRpcClient := rpcclient.NewConversationRpcClient(client, config.Share.RpcRegisterName.Conversation) groupRpcClient := rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) - historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgDatabase, &conversationRpcClient, &groupRpcClient) + historyCH, err := NewOnlineHistoryRedisConsumerHandler(&config.KafkaConfig, msgTransferDatabase, &conversationRpcClient, &groupRpcClient) if err != nil { return err } - historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(&config.KafkaConfig, msgDatabase) + historyMongoCH, err := NewOnlineHistoryMongoConsumerHandler(&config.KafkaConfig, msgTransferDatabase) if err != nil { return err } + msgTransfer := &MsgTransfer{ historyCH: historyCH, historyMongoCH: historyMongoCH, diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 8c1978d48b..b0078649cb 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,6 +18,10 @@ import ( "context" "encoding/json" "errors" + "strconv" + "strings" + "time" + "github.com/IBM/sarama" "github.com/go-redis/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/config" @@ -33,9 +37,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/utils/stringutil" "google.golang.org/protobuf/proto" - "strconv" - "strings" - "time" ) const ( @@ -56,19 +57,19 @@ type OnlineHistoryRedisConsumerHandler struct { redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage] - msgDatabase controller.CommonMsgDatabase + msgTransferDatabase controller.MsgTransferDatabase conversationRpcClient *rpcclient.ConversationRpcClient groupRpcClient *rpcclient.GroupRpcClient } -func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.CommonMsgDatabase, +func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase, conversationRpcClient *rpcclient.ConversationRpcClient, groupRpcClient *rpcclient.GroupRpcClient) (*OnlineHistoryRedisConsumerHandler, error) { historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToRedisGroupID, []string{kafkaConf.ToRedisTopic}, false) if err != nil { return nil, err } var och OnlineHistoryRedisConsumerHandler - och.msgDatabase = database + och.msgTransferDatabase = database b := batcher.New[sarama.ConsumerMessage]( batcher.WithSize(size), @@ -161,7 +162,7 @@ func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, return } for key, seq := range readSeq { - if err := och.msgDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { log.ZError(ctx, "set read seq to db error", err, "userID", key.userID, "conversationID", key.conversationID, "seq", seq) } } @@ -237,6 +238,11 @@ func (och *OnlineHistoryRedisConsumerHandler) categorizeMessageLists(totalMsgs [ } func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key, conversationID string, storageList, notStorageList []*ContextMsg) { + log.ZInfo(ctx, "handle storage msg") + for _, storageMsg := range storageList { + log.ZDebug(ctx, "handle storage msg", "msg", storageMsg.message.String()) + } + och.toPushTopic(ctx, key, conversationID, notStorageList) var storageMessageList []*sdkws.MsgData for _, msg := range storageList { @@ -244,21 +250,25 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } if len(storageMessageList) > 0 { msg := storageMessageList[0] - lastSeq, isNewConversation, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, isNewConversation, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil && !errors.Is(errs.Unwrap(err), redis.Nil) { log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) return } + log.ZInfo(ctx, "BatchInsertChat2Cache end") + if isNewConversation { switch msg.SessionType { case constant.ReadGroupChatType: - log.ZInfo(ctx, "group chat first create conversation", "conversationID", + log.ZDebug(ctx, "group chat first create conversation", "conversationID", conversationID) userIDs, err := och.groupRpcClient.GetGroupMemberIDs(ctx, msg.GroupID) if err != nil { log.ZWarn(ctx, "get group member ids error", err, "conversationID", conversationID) } else { + log.ZInfo(ctx, "GetGroupMemberIDs end") + if err := och.conversationRpcClient.GroupChatFirstCreateConversation(ctx, msg.GroupID, userIDs); err != nil { log.ZWarn(ctx, "single chat first create conversation error", err, @@ -277,13 +287,16 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } } - log.ZDebug(ctx, "success incr to next topic") - err = och.msgDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) + log.ZInfo(ctx, "success incr to next topic") + err = och.msgTransferDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) if err != nil { log.ZError(ctx, "Msg To MongoDB MQ error", err, "conversationID", conversationID, "storageList", storageMessageList, "lastSeq", lastSeq) } + log.ZInfo(ctx, "MsgToMongoMQ end") + och.toPushTopic(ctx, key, conversationID, storageList) + log.ZInfo(ctx, "toPushTopic end") } } @@ -295,14 +308,14 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con storageMessageList = append(storageMessageList, msg.message) } if len(storageMessageList) > 0 { - lastSeq, _, err := och.msgDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, _, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil { log.ZError(ctx, "notification batch insert to redis error", err, "conversationID", conversationID, "storageList", storageMessageList) return } log.ZDebug(ctx, "success to next topic", "conversationID", conversationID) - err = och.msgDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) + err = och.msgTransferDatabase.MsgToMongoMQ(ctx, key, conversationID, storageMessageList, lastSeq) if err != nil { log.ZError(ctx, "Msg To MongoDB MQ error", err, "conversationID", conversationID, "storageList", storageMessageList, "lastSeq", lastSeq) @@ -311,9 +324,10 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con } } -func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(_ context.Context, key, conversationID string, msgs []*ContextMsg) { +func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { - och.msgDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) + log.ZDebug(ctx, "push msg to topic", "msg", v.message.String()) + _, _, _ = och.msgTransferDatabase.MsgToPushMQ(v.ctx, key, conversationID, v.message) } } @@ -338,7 +352,7 @@ func (och *OnlineHistoryRedisConsumerHandler) Cleanup(_ sarama.ConsumerGroupSess func (och *OnlineHistoryRedisConsumerHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { // a instance in the consumer group - log.ZInfo(context.Background(), "online new session msg come", "highWaterMarkOffset", + log.ZDebug(context.Background(), "online new session msg come", "highWaterMarkOffset", claim.HighWaterMarkOffset(), "topic", claim.Topic(), "partition", claim.Partition()) och.redisMessageBatches.OnComplete = func(lastMessage *sarama.ConsumerMessage, totalCount int) { session.MarkMessage(lastMessage, "") diff --git a/internal/msgtransfer/online_msg_to_mongo_handler.go b/internal/msgtransfer/online_msg_to_mongo_handler.go index cea47fcd50..82002c26b9 100644 --- a/internal/msgtransfer/online_msg_to_mongo_handler.go +++ b/internal/msgtransfer/online_msg_to_mongo_handler.go @@ -16,6 +16,7 @@ package msgtransfer import ( "context" + "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" @@ -28,10 +29,10 @@ import ( type OnlineHistoryMongoConsumerHandler struct { historyConsumerGroup *kafka.MConsumerGroup - msgDatabase controller.CommonMsgDatabase + msgTransferDatabase controller.MsgTransferDatabase } -func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database controller.CommonMsgDatabase) (*OnlineHistoryMongoConsumerHandler, error) { +func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase) (*OnlineHistoryMongoConsumerHandler, error) { historyConsumerGroup, err := kafka.NewMConsumerGroup(kafkaConf.Build(), kafkaConf.ToMongoGroupID, []string{kafkaConf.ToMongoTopic}, true) if err != nil { return nil, err @@ -39,7 +40,7 @@ func NewOnlineHistoryMongoConsumerHandler(kafkaConf *config.Kafka, database cont mc := &OnlineHistoryMongoConsumerHandler{ historyConsumerGroup: historyConsumerGroup, - msgDatabase: database, + msgTransferDatabase: database, } return mc, nil } @@ -56,8 +57,8 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont log.ZError(ctx, "msgFromMQ.MsgData is empty", nil, "cMsg", cMsg) return } - log.ZInfo(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) - err = mc.msgDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) + log.ZDebug(ctx, "mongo consumer recv msg", "msgs", msgFromMQ.String()) + err = mc.msgTransferDatabase.BatchInsertChat2DB(ctx, msgFromMQ.ConversationID, msgFromMQ.MsgData, msgFromMQ.LastSeq) if err != nil { log.ZError( ctx, @@ -76,7 +77,7 @@ func (mc *OnlineHistoryMongoConsumerHandler) handleChatWs2Mongo(ctx context.Cont for _, msg := range msgFromMQ.MsgData { seqs = append(seqs, msg.Seq) } - err = mc.msgDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) + err = mc.msgTransferDatabase.DeleteMessagesFromCache(ctx, msgFromMQ.ConversationID, seqs) if err != nil { log.ZError( ctx, diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 028e7edd34..09831cabfa 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -17,6 +17,7 @@ package dummy import ( "context" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/tools/log" ) func NewClient() *Dummy { @@ -27,5 +28,6 @@ type Dummy struct { } func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { + log.ZDebug(ctx, "dummy push") return nil } diff --git a/internal/push/offlinepush/getui/push.go b/internal/push/offlinepush/getui/push.go index 27b19e8fe0..e266f9c464 100644 --- a/internal/push/offlinepush/getui/push.go +++ b/internal/push/offlinepush/getui/push.go @@ -18,11 +18,11 @@ import ( "context" "crypto/sha256" "encoding/hex" - "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "strconv" "sync" "time" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/tools/errs" @@ -91,6 +91,15 @@ func (g *Client) Push(ctx context.Context, userIDs []string, title, content stri for i, v := range s.GetSplitResult() { go func(index int, userIDs []string) { defer wg.Done() + for i := 0; i < len(userIDs); i += maxNum { + end := i + maxNum + if end > len(userIDs) { + end = len(userIDs) + } + if err = g.batchPush(ctx, token, userIDs[i:end], pushReq); err != nil { + log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) + } + } if err = g.batchPush(ctx, token, userIDs, pushReq); err != nil { log.ZError(ctx, "batchPush failed", err, "index", index, "token", token, "req", pushReq) } diff --git a/internal/push/offlinepush_handler.go b/internal/push/offlinepush_handler.go new file mode 100644 index 0000000000..bf69aed3e2 --- /dev/null +++ b/internal/push/offlinepush_handler.go @@ -0,0 +1,122 @@ +package push + +import ( + "context" + + "github.com/IBM/sarama" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/protocol/constant" + pbpush "github.com/openimsdk/protocol/push" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" + "github.com/openimsdk/tools/utils/jsonutil" + "google.golang.org/protobuf/proto" +) + +type OfflinePushConsumerHandler struct { + OfflinePushConsumerGroup *kafka.MConsumerGroup + offlinePusher offlinepush.OfflinePusher +} + +func NewOfflinePushConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher) (*OfflinePushConsumerHandler, error) { + var offlinePushConsumerHandler OfflinePushConsumerHandler + var err error + offlinePushConsumerHandler.offlinePusher = offlinePusher + offlinePushConsumerHandler.OfflinePushConsumerGroup, err = kafka.NewMConsumerGroup(config.KafkaConfig.Build(), config.KafkaConfig.ToOfflineGroupID, + []string{config.KafkaConfig.ToOfflinePushTopic}, true) + if err != nil { + return nil, err + } + return &offlinePushConsumerHandler, nil +} + +func (*OfflinePushConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } +func (*OfflinePushConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } +func (o *OfflinePushConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for msg := range claim.Messages() { + ctx := o.OfflinePushConsumerGroup.GetContextFromMsg(msg) + o.handleMsg2OfflinePush(ctx, msg.Value) + sess.MarkMessage(msg, "") + } + return nil +} + +func (o *OfflinePushConsumerHandler) handleMsg2OfflinePush(ctx context.Context, msg []byte) { + offlinePushMsg := pbpush.PushMsgReq{} + if err := proto.Unmarshal(msg, &offlinePushMsg); err != nil { + log.ZError(ctx, "offline push Unmarshal msg err", err, "msg", string(msg)) + return + } + if offlinePushMsg.MsgData == nil || offlinePushMsg.UserIDs == nil { + log.ZError(ctx, "offline push msg is empty", errs.New("offlinePushMsg is empty"), "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) + return + } + log.ZInfo(ctx, "receive to OfflinePush MQ", "userIDs", offlinePushMsg.UserIDs, "msg", offlinePushMsg.MsgData) + + err := o.offlinePushMsg(ctx, offlinePushMsg.MsgData, offlinePushMsg.UserIDs) + if err != nil { + log.ZWarn(ctx, "offline push failed", err, "msg", offlinePushMsg.String()) + } +} + +func (o *OfflinePushConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, content string, opts *options.Opts, err error) { + type AtTextElem struct { + Text string `json:"text,omitempty"` + AtUserList []string `json:"atUserList,omitempty"` + IsAtSelf bool `json:"isAtSelf"` + } + + opts = &options.Opts{Signal: &options.Signal{}} + if msg.OfflinePushInfo != nil { + opts.IOSBadgeCount = msg.OfflinePushInfo.IOSBadgeCount + opts.IOSPushSound = msg.OfflinePushInfo.IOSPushSound + opts.Ex = msg.OfflinePushInfo.Ex + } + + if msg.OfflinePushInfo != nil { + title = msg.OfflinePushInfo.Title + content = msg.OfflinePushInfo.Desc + } + if title == "" { + switch msg.ContentType { + case constant.Text: + fallthrough + case constant.Picture: + fallthrough + case constant.Voice: + fallthrough + case constant.Video: + fallthrough + case constant.File: + title = constant.ContentType2PushContent[int64(msg.ContentType)] + case constant.AtText: + ac := AtTextElem{} + _ = jsonutil.JsonStringToStruct(string(msg.Content), &ac) + case constant.SignalingNotification: + title = constant.ContentType2PushContent[constant.SignalMsg] + default: + title = constant.ContentType2PushContent[constant.Common] + } + } + if content == "" { + content = title + } + return +} + +func (o *OfflinePushConsumerHandler) offlinePushMsg(ctx context.Context, msg *sdkws.MsgData, offlinePushUserIDs []string) error { + title, content, opts, err := o.getOfflinePushInfos(msg) + if err != nil { + return err + } + err = o.offlinePusher.Push(ctx, offlinePushUserIDs, title, content, opts) + if err != nil { + prommetrics.MsgOfflinePushFailedCounter.Inc() + return err + } + return nil +} diff --git a/internal/push/onlinepusher.go b/internal/push/onlinepusher.go index d0c65e06bf..9521a84a07 100644 --- a/internal/push/onlinepusher.go +++ b/internal/push/onlinepusher.go @@ -27,12 +27,12 @@ func newEmptyOnlinePusher() *emptyOnlinePusher { func (emptyOnlinePusher) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) (wsResults []*msggateway.SingleMsgToUserResults, err error) { - log.ZWarn(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil) + log.ZInfo(ctx, "emptyOnlinePusher GetConnsAndOnlinePush", nil) return nil, nil } func (u emptyOnlinePusher) GetOnlinePushFailedUserIDs(ctx context.Context, msg *sdkws.MsgData, wsResults []*msggateway.SingleMsgToUserResults, pushToUserIDs *[]string) []string { - log.ZWarn(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil) + log.ZInfo(ctx, "emptyOnlinePusher GetOnlinePushFailedUserIDs", nil) return nil } diff --git a/internal/push/push.go b/internal/push/push.go index 1a04bbea26..850f91d22e 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -2,6 +2,7 @@ package push import ( "context" + "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" @@ -17,12 +18,12 @@ type pushServer struct { disCov discovery.SvcDiscoveryRegistry offlinePusher offlinepush.OfflinePusher pushCh *ConsumerHandler + offlinePushCh *OfflinePushConsumerHandler } type Config struct { RpcConfig config.Push RedisConfig config.Redis - MongodbConfig config.Mongo KafkaConfig config.Kafka NotificationConfig config.Notification Share config.Share @@ -55,18 +56,30 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - database := controller.NewPushDatabase(cacheModel) - consumer, err := NewConsumerHandler(config, offlinePusher, rdb, client) + database := controller.NewPushDatabase(cacheModel, &config.KafkaConfig) + + consumer, err := NewConsumerHandler(config, database, offlinePusher, rdb, client) + if err != nil { + return err + } + + offlinePushConsumer, err := NewOfflinePushConsumerHandler(config, offlinePusher) if err != nil { return err } + pbpush.RegisterPushMsgServiceServer(server, &pushServer{ database: database, disCov: client, offlinePusher: offlinePusher, pushCh: consumer, + offlinePushCh: offlinePushConsumer, }) + go consumer.pushConsumerGroup.RegisterHandleAndConsumer(ctx, consumer) + + go offlinePushConsumer.OfflinePushConsumerGroup.RegisterHandleAndConsumer(ctx, offlinePushConsumer) + return nil } diff --git a/internal/push/push_handler.go b/internal/push/push_handler.go index 79d3a92966..4ecf20de52 100644 --- a/internal/push/push_handler.go +++ b/internal/push/push_handler.go @@ -1,33 +1,20 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package push import ( "context" "encoding/json" + "github.com/IBM/sarama" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" "github.com/openimsdk/open-im-server/v3/pkg/rpccache" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" - pbchat "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/msggateway" pbpush "github.com/openimsdk/protocol/push" "github.com/openimsdk/protocol/sdkws" @@ -40,12 +27,16 @@ import ( "github.com/openimsdk/tools/utils/timeutil" "github.com/redis/go-redis/v9" "google.golang.org/protobuf/proto" + "math/rand" + "strconv" + "time" ) type ConsumerHandler struct { pushConsumerGroup *kafka.MConsumerGroup offlinePusher offlinepush.OfflinePusher onlinePusher OnlinePusher + pushDatabase controller.PushDatabase onlineCache *rpccache.OnlineCache groupLocalCache *rpccache.GroupLocalCache conversationLocalCache *rpccache.ConversationLocalCache @@ -56,7 +47,7 @@ type ConsumerHandler struct { config *Config } -func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, +func NewConsumerHandler(config *Config, database controller.PushDatabase, offlinePusher offlinepush.OfflinePusher, rdb redis.UniversalClient, client discovery.SvcDiscoveryRegistry) (*ConsumerHandler, error) { var consumerHandler ConsumerHandler var err error @@ -65,7 +56,9 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, if err != nil { return nil, err } + userRpcClient := rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID) + consumerHandler.offlinePusher = offlinePusher consumerHandler.onlinePusher = NewOnlinePusher(client, config) consumerHandler.groupRpcClient = rpcclient.NewGroupRpcClient(client, config.Share.RpcRegisterName.Group) @@ -75,42 +68,45 @@ func NewConsumerHandler(config *Config, offlinePusher offlinepush.OfflinePusher, consumerHandler.conversationLocalCache = rpccache.NewConversationLocalCache(consumerHandler.conversationRpcClient, &config.LocalCacheConfig, rdb) consumerHandler.webhookClient = webhook.NewWebhookClient(config.WebhooksConfig.URL) consumerHandler.config = config - consumerHandler.onlineCache = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, nil) + consumerHandler.pushDatabase = database + consumerHandler.onlineCache, err = rpccache.NewOnlineCache(userRpcClient, consumerHandler.groupLocalCache, rdb, config.RpcConfig.FullUserCache, nil) + if err != nil { + return nil, err + } return &consumerHandler, nil } func (c *ConsumerHandler) handleMs2PsChat(ctx context.Context, msg []byte) { - msgFromMQ := pbchat.PushMsgDataToMQ{} + msgFromMQ := pbpush.PushMsgReq{} if err := proto.Unmarshal(msg, &msgFromMQ); err != nil { log.ZError(ctx, "push Unmarshal msg err", err, "msg", string(msg)) return } - pbData := &pbpush.PushMsgReq{ - MsgData: msgFromMQ.MsgData, - ConversationID: msgFromMQ.ConversationID, - } + sec := msgFromMQ.MsgData.SendTime / 1000 nowSec := timeutil.GetCurrentTimestampBySecond() if nowSec-sec > 10 { - log.ZWarn(ctx, "long time push msg", nil, "msg", pbData.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) + prommetrics.MsgLoneTimePushCounter.Inc() + log.ZWarn(ctx, "it’s been a while since the message was sent", nil, "msg", msgFromMQ.String(), "sec", sec, "nowSec", nowSec, "nowSec-sec", nowSec-sec) } var err error + switch msgFromMQ.MsgData.SessionType { case constant.ReadGroupChatType: - err = c.Push2Group(ctx, pbData.MsgData.GroupID, pbData.MsgData) + err = c.Push2Group(ctx, msgFromMQ.MsgData.GroupID, msgFromMQ.MsgData) default: var pushUserIDList []string - isSenderSync := datautil.GetSwitchFromOptions(pbData.MsgData.Options, constant.IsSenderSync) - if !isSenderSync || pbData.MsgData.SendID == pbData.MsgData.RecvID { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID) + isSenderSync := datautil.GetSwitchFromOptions(msgFromMQ.MsgData.Options, constant.IsSenderSync) + if !isSenderSync || msgFromMQ.MsgData.SendID == msgFromMQ.MsgData.RecvID { + pushUserIDList = append(pushUserIDList, msgFromMQ.MsgData.RecvID) } else { - pushUserIDList = append(pushUserIDList, pbData.MsgData.RecvID, pbData.MsgData.SendID) + pushUserIDList = append(pushUserIDList, msgFromMQ.MsgData.RecvID, msgFromMQ.MsgData.SendID) } - err = c.Push2User(ctx, pushUserIDList, pbData.MsgData) + err = c.Push2User(ctx, pushUserIDList, msgFromMQ.MsgData) } if err != nil { - log.ZWarn(ctx, "push failed", err, "msg", pbData.String()) + log.ZWarn(ctx, "push failed", err, "msg", msgFromMQ.String()) } } @@ -119,6 +115,14 @@ func (*ConsumerHandler) Setup(sarama.ConsumerGroupSession) error { return nil } func (*ConsumerHandler) Cleanup(sarama.ConsumerGroupSession) error { return nil } func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + c.onlineCache.Lock.Lock() + for c.onlineCache.CurrentPhase.Load() < rpccache.DoSubscribeOver { + c.onlineCache.Cond.Wait() + } + c.onlineCache.Lock.Unlock() + ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10)) + log.ZInfo(ctx, "begin consume messages") + for msg := range claim.Messages() { ctx := c.pushConsumerGroup.GetContextFromMsg(msg) c.handleMs2PsChat(ctx, msg.Value) @@ -129,20 +133,27 @@ func (c *ConsumerHandler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim s // Push2User Suitable for two types of conversations, one is SingleChatType and the other is NotificationChatType. func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg *sdkws.MsgData) (err error) { - log.ZDebug(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) + log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "userIDs", userIDs, "msg", msg.String()) + defer func(duration time.Time) { + t := time.Since(duration) + log.ZInfo(ctx, "Get msg from msg_transfer And push msg", "msg", msg.String(), "time cost", t) + }(time.Now()) if err := c.webhookBeforeOnlinePush(ctx, &c.config.WebhooksConfig.BeforeOnlinePush, userIDs, msg); err != nil { return err } + log.ZInfo(ctx, "webhookBeforeOnlinePush end") + wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, userIDs) if err != nil { return err } - log.ZDebug(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) + log.ZInfo(ctx, "single and notification push result", "result", wsResults, "msg", msg, "push_to_userID", userIDs) if !c.shouldPushOffline(ctx, msg) { return nil } + log.ZInfo(ctx, "shouldPushOffline end") for _, v := range wsResults { //message sender do not need offline push @@ -161,7 +172,7 @@ func (c *ConsumerHandler) Push2User(ctx context.Context, userIDs []string, msg * offlinePushUserID, msg, nil); err != nil { return err } - + log.ZInfo(ctx, "webhookBeforeOfflinePush end") err = c.offlinePushMsg(ctx, msg, offlinePushUserID) if err != nil { log.ZWarn(ctx, "offlinePushMsg failed", err, "offlinePushUserID", offlinePushUserID, "msg", msg) @@ -183,21 +194,11 @@ func (c *ConsumerHandler) shouldPushOffline(_ context.Context, msg *sdkws.MsgDat } func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws.MsgData, pushToUserIDs []string) ([]*msggateway.SingleMsgToUserResults, error) { - var ( - onlineUserIDs []string - offlineUserIDs []string - ) - for _, userID := range pushToUserIDs { - online, err := c.onlineCache.GetUserOnline(ctx, userID) - if err != nil { - return nil, err - } - if online { - onlineUserIDs = append(onlineUserIDs, userID) - } else { - offlineUserIDs = append(offlineUserIDs, userID) - } + onlineUserIDs, offlineUserIDs, err := c.onlineCache.GetUsersOnline(ctx, pushToUserIDs) + if err != nil { + return nil, err } + log.ZDebug(ctx, "GetConnsAndOnlinePush online cache", "sendID", msg.SendID, "recvID", msg.RecvID, "groupID", msg.GroupID, "sessionType", msg.SessionType, "clientMsgID", msg.ClientMsgID, "serverMsgID", msg.ServerMsgID, "offlineUserIDs", offlineUserIDs, "onlineUserIDs", onlineUserIDs) var result []*msggateway.SingleMsgToUserResults if len(onlineUserIDs) > 0 { @@ -216,57 +217,70 @@ func (c *ConsumerHandler) GetConnsAndOnlinePush(ctx context.Context, msg *sdkws. } func (c *ConsumerHandler) Push2Group(ctx context.Context, groupID string, msg *sdkws.MsgData) (err error) { - log.ZDebug(ctx, "Get group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) + log.ZInfo(ctx, "Get group msg from msg_transfer and push msg", "msg", msg.String(), "groupID", groupID) + defer func(duration time.Time) { + t := time.Since(duration) + log.ZInfo(ctx, "Get group msg from msg_transfer and push msg end", "msg", msg.String(), "groupID", groupID, "time cost", t) + }(time.Now()) var pushToUserIDs []string if err = c.webhookBeforeGroupOnlinePush(ctx, &c.config.WebhooksConfig.BeforeGroupOnlinePush, groupID, msg, &pushToUserIDs); err != nil { return err } + log.ZInfo(ctx, "webhookBeforeGroupOnlinePush end") err = c.groupMessagesHandler(ctx, groupID, &pushToUserIDs, msg) if err != nil { return err } + log.ZInfo(ctx, "groupMessagesHandler end") wsResults, err := c.GetConnsAndOnlinePush(ctx, msg, pushToUserIDs) if err != nil { return err } - log.ZDebug(ctx, "group push result", "result", wsResults, "msg", msg) + log.ZInfo(ctx, "group push result", "result", wsResults, "msg", msg) if !c.shouldPushOffline(ctx, msg) { return nil } needOfflinePushUserIDs := c.onlinePusher.GetOnlinePushFailedUserIDs(ctx, msg, wsResults, &pushToUserIDs) - + log.ZInfo(ctx, "GetOnlinePushFailedUserIDs end") //filter some user, like don not disturb or don't need offline push etc. needOfflinePushUserIDs, err = c.filterGroupMessageOfflinePush(ctx, groupID, msg, needOfflinePushUserIDs) if err != nil { return err } + log.ZInfo(ctx, "filterGroupMessageOfflinePush end") + // Use offline push messaging if len(needOfflinePushUserIDs) > 0 { - var offlinePushUserIDs []string - err = c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserIDs, msg, &offlinePushUserIDs) - if err != nil { - return err - } - - if len(offlinePushUserIDs) > 0 { - needOfflinePushUserIDs = offlinePushUserIDs - } + c.asyncOfflinePush(ctx, needOfflinePushUserIDs, msg) + } - err = c.offlinePushMsg(ctx, msg, needOfflinePushUserIDs) - if err != nil { - log.ZWarn(ctx, "offlinePushMsg failed", err, "groupID", groupID, "msg", msg) - return nil - } + return nil +} +func (c *ConsumerHandler) asyncOfflinePush(ctx context.Context, needOfflinePushUserIDs []string, msg *sdkws.MsgData) { + var offlinePushUserIDs []string + err := c.webhookBeforeOfflinePush(ctx, &c.config.WebhooksConfig.BeforeOfflinePush, needOfflinePushUserIDs, msg, &offlinePushUserIDs) + if err != nil { + log.ZWarn(ctx, "webhookBeforeOfflinePush failed", err, "msg", msg) + return } - return nil + if len(offlinePushUserIDs) > 0 { + needOfflinePushUserIDs = offlinePushUserIDs + } + if err := c.pushDatabase.MsgToOfflinePushMQ(ctx, conversationutil.GenConversationUniqueKeyForSingle(msg.SendID, msg.RecvID), needOfflinePushUserIDs, msg); err != nil { + log.ZError(ctx, "Msg To OfflinePush MQ error", err, "needOfflinePushUserIDs", + needOfflinePushUserIDs, "msg", msg) + prommetrics.SingleChatMsgProcessFailedCounter.Inc() + return + } } + func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID string, pushToUserIDs *[]string, msg *sdkws.MsgData) (err error) { if len(*pushToUserIDs) == 0 { *pushToUserIDs, err = c.groupLocalCache.GetGroupMemberIDs(ctx, groupID) @@ -300,7 +314,7 @@ func (c *ConsumerHandler) groupMessagesHandler(ctx context.Context, groupID stri if unmarshalNotificationElem(msg.Content, &tips) != nil { return err } - log.ZInfo(ctx, "GroupDismissedNotificationInfo****", "groupID", groupID, "num", len(*pushToUserIDs), "list", pushToUserIDs) + log.ZDebug(ctx, "GroupDismissedNotificationInfo****", "groupID", groupID, "num", len(*pushToUserIDs), "list", pushToUserIDs) if len(c.config.Share.IMAdminUserID) > 0 { ctx = mcontext.WithOpUserIDContext(ctx, c.config.Share.IMAdminUserID[0]) } @@ -384,6 +398,7 @@ func (c *ConsumerHandler) getOfflinePushInfos(msg *sdkws.MsgData) (title, conten } return } + func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, groupID string, userIDs []string) error { conversationID := msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, groupID) maxSeq, err := c.msgRpcClient.GetConversationMaxSeq(ctx, conversationID) @@ -392,6 +407,7 @@ func (c *ConsumerHandler) DeleteMemberAndSetConversationSeq(ctx context.Context, } return c.conversationRpcClient.SetConversationMaxSeq(ctx, userIDs, conversationID, maxSeq) } + func unmarshalNotificationElem(bytes []byte, t any) error { var notification sdkws.NotificationElem if err := json.Unmarshal(bytes, ¬ification); err != nil { diff --git a/internal/rpc/conversation/conversaion.go b/internal/rpc/conversation/conversaion.go index 6f77164e38..66e85cb8f7 100644 --- a/internal/rpc/conversation/conversaion.go +++ b/internal/rpc/conversation/conversaion.go @@ -278,8 +278,8 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver if req.Conversation.MsgDestructTime != nil { m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value } - if req.Conversation.MsgDestructTime != nil { - m["msg_destruct_time"] = req.Conversation.MsgDestructTime.Value + if req.Conversation.IsMsgDestruct != nil { + m["is_msg_destruct"] = req.Conversation.IsMsgDestruct.Value } if req.Conversation.BurnDuration != nil { m["burn_duration"] = req.Conversation.BurnDuration.Value diff --git a/internal/rpc/group/callback.go b/internal/rpc/group/callback.go index 5e3dc9b9c3..ef391bd70c 100644 --- a/internal/rpc/group/callback.go +++ b/internal/rpc/group/callback.go @@ -359,73 +359,73 @@ func (s *groupServer) webhookAfterSetGroupInfo(ctx context.Context, after *confi s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoResp{}, after) } -func (s *groupServer) webhookBeforeSetGroupInfoEX(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoEXReq) error { +func (s *groupServer) webhookBeforeSetGroupInfoEx(ctx context.Context, before *config.BeforeConfig, req *group.SetGroupInfoExReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { - cbReq := &callbackstruct.CallbackBeforeSetGroupInfoEXReq{ - CallbackCommand: callbackstruct.CallbackBeforeSetGroupInfoCommand, - GroupID: req.GroupInfoForSetEX.GroupID, - GroupName: req.GroupInfoForSetEX.GroupName, - Notification: req.GroupInfoForSetEX.Notification, - Introduction: req.GroupInfoForSetEX.Introduction, - FaceURL: req.GroupInfoForSetEX.FaceURL, + cbReq := &callbackstruct.CallbackBeforeSetGroupInfoExReq{ + CallbackCommand: callbackstruct.CallbackBeforeSetGroupInfoExCommand, + GroupID: req.GroupID, + GroupName: req.GroupName, + Notification: req.Notification, + Introduction: req.Introduction, + FaceURL: req.FaceURL, } - if req.GroupInfoForSetEX.Ex != nil { - cbReq.Ex = req.GroupInfoForSetEX.Ex + if req.Ex != nil { + cbReq.Ex = req.Ex } log.ZDebug(ctx, "debug CallbackBeforeSetGroupInfoEX", "ex", cbReq.Ex) - if req.GroupInfoForSetEX.NeedVerification != nil { - cbReq.NeedVerification = req.GroupInfoForSetEX.NeedVerification + if req.NeedVerification != nil { + cbReq.NeedVerification = req.NeedVerification } - if req.GroupInfoForSetEX.LookMemberInfo != nil { - cbReq.LookMemberInfo = req.GroupInfoForSetEX.LookMemberInfo + if req.LookMemberInfo != nil { + cbReq.LookMemberInfo = req.LookMemberInfo } - if req.GroupInfoForSetEX.ApplyMemberFriend != nil { - cbReq.ApplyMemberFriend = req.GroupInfoForSetEX.ApplyMemberFriend + if req.ApplyMemberFriend != nil { + cbReq.ApplyMemberFriend = req.ApplyMemberFriend } - resp := &callbackstruct.CallbackBeforeSetGroupInfoEXResp{} + resp := &callbackstruct.CallbackBeforeSetGroupInfoExResp{} if err := s.webhookClient.SyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, before); err != nil { return err } - datautil.NotNilReplace(&req.GroupInfoForSetEX.GroupID, &resp.GroupID) - datautil.NotNilReplace(&req.GroupInfoForSetEX.GroupName, &resp.GroupName) - datautil.NotNilReplace(&req.GroupInfoForSetEX.FaceURL, &resp.FaceURL) - datautil.NotNilReplace(&req.GroupInfoForSetEX.Introduction, &resp.Introduction) - datautil.NotNilReplace(&req.GroupInfoForSetEX.Ex, &resp.Ex) - datautil.NotNilReplace(&req.GroupInfoForSetEX.NeedVerification, &resp.NeedVerification) - datautil.NotNilReplace(&req.GroupInfoForSetEX.LookMemberInfo, &resp.LookMemberInfo) - datautil.NotNilReplace(&req.GroupInfoForSetEX.ApplyMemberFriend, &resp.ApplyMemberFriend) + datautil.NotNilReplace(&req.GroupID, &resp.GroupID) + datautil.NotNilReplace(&req.GroupName, &resp.GroupName) + datautil.NotNilReplace(&req.FaceURL, &resp.FaceURL) + datautil.NotNilReplace(&req.Introduction, &resp.Introduction) + datautil.NotNilReplace(&req.Ex, &resp.Ex) + datautil.NotNilReplace(&req.NeedVerification, &resp.NeedVerification) + datautil.NotNilReplace(&req.LookMemberInfo, &resp.LookMemberInfo) + datautil.NotNilReplace(&req.ApplyMemberFriend, &resp.ApplyMemberFriend) return nil }) } -func (s *groupServer) webhookAfterSetGroupInfoEX(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoEXReq) { - cbReq := &callbackstruct.CallbackAfterSetGroupInfoEXReq{ - CallbackCommand: callbackstruct.CallbackAfterSetGroupInfoCommand, - GroupID: req.GroupInfoForSetEX.GroupID, - GroupName: req.GroupInfoForSetEX.GroupName, - Notification: req.GroupInfoForSetEX.Notification, - Introduction: req.GroupInfoForSetEX.Introduction, - FaceURL: req.GroupInfoForSetEX.FaceURL, +func (s *groupServer) webhookAfterSetGroupInfoEx(ctx context.Context, after *config.AfterConfig, req *group.SetGroupInfoExReq) { + cbReq := &callbackstruct.CallbackAfterSetGroupInfoExReq{ + CallbackCommand: callbackstruct.CallbackAfterSetGroupInfoExCommand, + GroupID: req.GroupID, + GroupName: req.GroupName, + Notification: req.Notification, + Introduction: req.Introduction, + FaceURL: req.FaceURL, } - if req.GroupInfoForSetEX.Ex != nil { - cbReq.Ex = req.GroupInfoForSetEX.Ex + if req.Ex != nil { + cbReq.Ex = req.Ex } - if req.GroupInfoForSetEX.NeedVerification != nil { - cbReq.NeedVerification = req.GroupInfoForSetEX.NeedVerification + if req.NeedVerification != nil { + cbReq.NeedVerification = req.NeedVerification } - if req.GroupInfoForSetEX.LookMemberInfo != nil { - cbReq.LookMemberInfo = req.GroupInfoForSetEX.LookMemberInfo + if req.LookMemberInfo != nil { + cbReq.LookMemberInfo = req.LookMemberInfo } - if req.GroupInfoForSetEX.ApplyMemberFriend != nil { - cbReq.ApplyMemberFriend = req.GroupInfoForSetEX.ApplyMemberFriend + if req.ApplyMemberFriend != nil { + cbReq.ApplyMemberFriend = req.ApplyMemberFriend } - s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoEXResp{}, after) + s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, &callbackstruct.CallbackAfterSetGroupInfoExResp{}, after) } diff --git a/internal/rpc/group/db_map.go b/internal/rpc/group/db_map.go index 08895f9c5f..2f6791d5e0 100644 --- a/internal/rpc/group/db_map.go +++ b/internal/rpc/group/db_map.go @@ -20,6 +20,7 @@ import ( pbgroup "github.com/openimsdk/protocol/group" "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/mcontext" ) @@ -54,11 +55,11 @@ func UpdateGroupInfoMap(ctx context.Context, group *sdkws.GroupInfoForSet) map[s return m } -func UpdateGroupInfoEXMap(ctx context.Context, group *sdkws.GroupInfoForSetEX) map[string]any { +func UpdateGroupInfoExMap(ctx context.Context, group *pbgroup.SetGroupInfoExReq) (map[string]any, error) { m := make(map[string]any) - if group.GroupName != "" { - m["group_name"] = group.GroupName + if group.GroupName != nil && group.GroupName.Value != "" { + return nil, errs.ErrArgs.WrapMsg("group name is empty") } if group.Notification != nil { m["notification"] = group.Notification.Value @@ -84,7 +85,7 @@ func UpdateGroupInfoEXMap(ctx context.Context, group *sdkws.GroupInfoForSetEX) m m["ex"] = group.Ex.Value } - return m + return m, nil } func UpdateGroupStatusMap(status int) map[string]any { diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 9589ed2494..bb07834604 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -304,6 +304,13 @@ func (g *groupServer) CreateGroup(ctx context.Context, req *pbgroup.CreateGroupR } g.notification.GroupCreatedNotification(ctx, tips) + if req.GroupInfo.Notification != "" { + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{ + Group: tips.Group, + OpUser: tips.OpUser, + }) + } + reqCallBackAfter := &pbgroup.CreateGroupReq{ MemberUserIDs: userIDs, GroupInfo: resp.GroupInfo, @@ -833,7 +840,7 @@ func (g *groupServer) GroupApplicationResponse(ctx context.Context, req *pbgroup if member == nil { log.ZDebug(ctx, "GroupApplicationResponse", "member is nil") } else { - if err = g.notification.MemberEnterNotification(ctx, req.GroupID, req.FromUserID); err != nil { + if err = g.notification.GroupApplicationAgreeMemberEnterNotification(ctx, req.GroupID, groupRequest.InviterUserID, req.FromUserID); err != nil { return nil, err } } @@ -1028,12 +1035,12 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf } resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSet.GroupID}) if err != nil { - log.ZWarn(ctx, "GetGroupMemberIDs", err) + log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) return } conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { - log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation) + log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) } }() g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) @@ -1051,13 +1058,13 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf return &pbgroup.SetGroupInfoResp{}, nil } -func (g *groupServer) SetGroupInfoEX(ctx context.Context, req *pbgroup.SetGroupInfoEXReq) (*pbgroup.SetGroupInfoEXResp, error) { +func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupInfoExReq) (*pbgroup.SetGroupInfoExResp, error) { var opMember *model.GroupMember if !authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) { var err error - opMember, err = g.db.TakeGroupMember(ctx, req.GroupInfoForSetEX.GroupID, mcontext.GetOpUserID(ctx)) + opMember, err = g.db.TakeGroupMember(ctx, req.GroupID, mcontext.GetOpUserID(ctx)) if err != nil { return nil, err } @@ -1071,11 +1078,11 @@ func (g *groupServer) SetGroupInfoEX(ctx context.Context, req *pbgroup.SetGroupI } } - if err := g.webhookBeforeSetGroupInfoEX(ctx, &g.config.WebhooksConfig.BeforeSetGroupInfoEX, req); err != nil && err != servererrs.ErrCallbackContinue { + if err := g.webhookBeforeSetGroupInfoEx(ctx, &g.config.WebhooksConfig.BeforeSetGroupInfoEx, req); err != nil && err != servererrs.ErrCallbackContinue { return nil, err } - group, err := g.db.TakeGroup(ctx, req.GroupInfoForSetEX.GroupID) + group, err := g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -1097,16 +1104,20 @@ func (g *groupServer) SetGroupInfoEX(ctx context.Context, req *pbgroup.SetGroupI return nil, err } - updatedData := UpdateGroupInfoEXMap(ctx, req.GroupInfoForSetEX) + updatedData, err := UpdateGroupInfoExMap(ctx, req) if len(updatedData) == 0 { - return &pbgroup.SetGroupInfoEXResp{}, nil + return &pbgroup.SetGroupInfoExResp{}, nil + } + + if err != nil { + return nil, err } if err := g.db.UpdateGroup(ctx, group.GroupID, updatedData); err != nil { return nil, err } - group, err = g.db.TakeGroup(ctx, req.GroupInfoForSetEX.GroupID) + group, err = g.db.TakeGroup(ctx, req.GroupID) if err != nil { return nil, err } @@ -1122,43 +1133,46 @@ func (g *groupServer) SetGroupInfoEX(ctx context.Context, req *pbgroup.SetGroupI } num := len(updatedData) - if req.GroupInfoForSetEX.Notification != nil { + if req.Notification != nil { num-- - func() { - conversation := &pbconversation.ConversationReq{ - ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSetEX.GroupID), - ConversationType: constant.ReadGroupChatType, - GroupID: req.GroupInfoForSetEX.GroupID, - } + if req.Notification.Value != "" { + func() { + conversation := &pbconversation.ConversationReq{ + ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupID), + ConversationType: constant.ReadGroupChatType, + GroupID: req.GroupID, + } - resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupInfoForSetEX.GroupID}) - if err != nil { - log.ZWarn(ctx, "GetGroupMemberIDs", err) - return - } + resp, err := g.GetGroupMemberUserIDs(ctx, &pbgroup.GetGroupMemberUserIDsReq{GroupID: req.GroupID}) + if err != nil { + log.ZWarn(ctx, "GetGroupMemberIDs is failed.", err) + return + } - conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} + conversation.GroupAtType = &wrapperspb.Int32Value{Value: constant.GroupNotification} - if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { - log.ZWarn(ctx, "SetConversations", err, resp.UserIDs, conversation) - } - }() + if err := g.conversationRpcClient.SetConversations(ctx, resp.UserIDs, conversation); err != nil { + log.ZWarn(ctx, "SetConversations", err, "UserIDs", resp.UserIDs, "conversation", conversation) + } + }() - g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + g.notification.GroupInfoSetAnnouncementNotification(ctx, &sdkws.GroupInfoSetAnnouncementTips{Group: tips.Group, OpUser: tips.OpUser}) + } } - if req.GroupInfoForSetEX.GroupName != "" { - num-- + if req.GroupName != nil { + num-- g.notification.GroupInfoSetNameNotification(ctx, &sdkws.GroupInfoSetNameTips{Group: tips.Group, OpUser: tips.OpUser}) } + if num > 0 { g.notification.GroupInfoSetNotification(ctx, tips) } - g.webhookAfterSetGroupInfoEX(ctx, &g.config.WebhooksConfig.AfterSetGroupInfoEX, req) + g.webhookAfterSetGroupInfoEx(ctx, &g.config.WebhooksConfig.AfterSetGroupInfoEx, req) - return &pbgroup.SetGroupInfoEXResp{}, nil + return &pbgroup.SetGroupInfoExResp{}, nil } func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.TransferGroupOwnerReq) (*pbgroup.TransferGroupOwnerResp, error) { diff --git a/internal/rpc/group/notification.go b/internal/rpc/group/notification.go index 4a69b6aed2..64e922fe2b 100644 --- a/internal/rpc/group/notification.go +++ b/internal/rpc/group/notification.go @@ -16,27 +16,28 @@ package group import ( "context" + "errors" "fmt" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" + "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/versionctx" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" - "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" - "github.com/openimsdk/protocol/msg" - - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" + "github.com/openimsdk/open-im-server/v3/pkg/rpcclient/notification" "github.com/openimsdk/protocol/constant" pbgroup "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/tools/utils/stringutil" + "go.mongodb.org/mongo-driver/mongo" ) // GroupApplicationReceiver @@ -227,10 +228,13 @@ func (g *GroupNotificationSender) groupMemberDB2PB(member *model.GroupMember, ap } */ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws.GroupMemberFullInfo, groupID string) (err error) { + return g.fillOpUserByUserID(ctx, mcontext.GetOpUserID(ctx), opUser, groupID) +} + +func (g *GroupNotificationSender) fillOpUserByUserID(ctx context.Context, userID string, opUser **sdkws.GroupMemberFullInfo, groupID string) error { if opUser == nil { return errs.ErrInternalServer.WrapMsg("**sdkws.GroupMemberFullInfo is nil") } - userID := mcontext.GetOpUserID(ctx) if groupID != "" { if authverify.IsManagerUserID(userID, g.config.Share.IMAdminUserID) { *opUser = &sdkws.GroupMemberFullInfo{ @@ -243,7 +247,7 @@ func (g *GroupNotificationSender) fillOpUser(ctx context.Context, opUser **sdkws member, err := g.db.TakeGroupMember(ctx, groupID, userID) if err == nil { *opUser = g.groupMemberDB2PB(member, 0) - } else if !errs.ErrRecordNotFound.Is(err) { + } else if !(errors.Is(err, mongo.ErrNoDocuments) || errs.ErrRecordNotFound.Is(err)) { return err } } @@ -509,7 +513,7 @@ func (g *GroupNotificationSender) MemberKickedNotification(ctx context.Context, g.Notification(ctx, mcontext.GetOpUserID(ctx), tips.Group.GroupID, constant.MemberKickedNotification, tips) } -func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID ...string) error { +func (g *GroupNotificationSender) GroupApplicationAgreeMemberEnterNotification(ctx context.Context, groupID string, invitedOpUserID string, entrantUserID ...string) error { var err error defer func() { if err != nil { @@ -546,15 +550,32 @@ func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, g return err } - tips := &sdkws.MemberInvitedTips{Group: group, InvitedUserList: users} - if err = g.fillOpUser(ctx, &tips.OpUser, tips.Group.GroupID); err != nil { + tips := &sdkws.MemberInvitedTips{ + Group: group, + InvitedUserList: users, + } + opUserID := mcontext.GetOpUserID(ctx) + if err = g.fillOpUserByUserID(ctx, opUserID, &tips.OpUser, tips.Group.GroupID); err != nil { return nil } + switch { + case invitedOpUserID == "": + case invitedOpUserID == opUserID: + tips.InviterUser = tips.OpUser + default: + if err = g.fillOpUserByUserID(ctx, invitedOpUserID, &tips.InviterUser, tips.Group.GroupID); err != nil { + return err + } + } g.setVersion(ctx, &tips.GroupMemberVersion, &tips.GroupMemberVersionID, database.GroupMemberVersionName, tips.Group.GroupID) g.Notification(ctx, mcontext.GetOpUserID(ctx), group.GroupID, constant.MemberInvitedNotification, tips) return nil } +func (g *GroupNotificationSender) MemberEnterNotification(ctx context.Context, groupID string, entrantUserID ...string) error { + return g.GroupApplicationAgreeMemberEnterNotification(ctx, groupID, "", entrantUserID...) +} + func (g *GroupNotificationSender) GroupDismissedNotification(ctx context.Context, tips *sdkws.GroupDismissedTips) { var err error defer func() { diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 4ffa1f43ee..c5bd36b445 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -67,7 +67,7 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return nil, err } - log.ZInfo(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) + log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) return &msg.ClearMsgResp{}, nil } diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index f5b5ebda53..2f77881673 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -16,16 +16,16 @@ package msg import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" - "github.com/openimsdk/tools/utils/datautil" - "github.com/openimsdk/tools/utils/timeutil" "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/msgprocessor" + "github.com/openimsdk/open-im-server/v3/pkg/util/conversationutil" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/tools/utils/timeutil" ) func (m *msgServer) PullMessageBySeqs(ctx context.Context, req *sdkws.PullMessageBySeqsReq) (*sdkws.PullMessageBySeqsResp, error) { @@ -86,6 +86,35 @@ func (m *msgServer) PullMessageBySeqs(ctx context.Context, req *sdkws.PullMessag return resp, nil } +func (m *msgServer) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { + resp := &msg.GetSeqMessageResp{ + Msgs: make(map[string]*sdkws.PullMsgs), + NotificationMsgs: make(map[string]*sdkws.PullMsgs), + } + for _, conv := range req.Conversations { + _, _, msgs, err := m.MsgDatabase.GetMsgBySeqs(ctx, req.UserID, conv.ConversationID, conv.Seqs) + if err != nil { + return nil, err + } + var pullMsgs *sdkws.PullMsgs + if ok := false; conversationutil.IsNotificationConversationID(conv.ConversationID) { + pullMsgs, ok = resp.NotificationMsgs[conv.ConversationID] + if !ok { + pullMsgs = &sdkws.PullMsgs{} + resp.NotificationMsgs[conv.ConversationID] = pullMsgs + } + } else { + pullMsgs, ok = resp.Msgs[conv.ConversationID] + if !ok { + pullMsgs = &sdkws.PullMsgs{} + resp.Msgs[conv.ConversationID] = pullMsgs + } + } + pullMsgs.Msgs = append(pullMsgs.Msgs, msgs...) + } + return resp, nil +} + func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sdkws.GetMaxSeqResp, error) { if err := authverify.CheckAccessV3(ctx, req.UserID, m.config.Share.IMAdminUserID); err != nil { return nil, err @@ -104,13 +133,20 @@ func (m *msgServer) GetMaxSeq(ctx context.Context, req *sdkws.GetMaxSeqReq) (*sd log.ZWarn(ctx, "GetMaxSeqs error", err, "conversationIDs", conversationIDs, "maxSeqs", maxSeqs) return nil, err } + // avoid pulling messages from sessions with a large number of max seq values of 0 + for conversationID, seq := range maxSeqs { + if seq == 0 { + delete(maxSeqs, conversationID) + } + } resp := new(sdkws.GetMaxSeqResp) resp.MaxSeqs = maxSeqs return resp, nil } func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq) (resp *msg.SearchMessageResp, err error) { - var chatLogs []*sdkws.MsgData + // var chatLogs []*sdkws.MsgData + var chatLogs []*msg.SearchedMsgData var total int64 resp = &msg.SearchMessageResp{} if total, chatLogs, err = m.MsgDatabase.SearchMessage(ctx, req); err != nil { @@ -125,17 +161,19 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq recvMap = make(map[string]string) groupMap = make(map[string]*sdkws.GroupInfo) ) + for _, chatLog := range chatLogs { - if chatLog.SenderNickname == "" { - sendIDs = append(sendIDs, chatLog.SendID) + if chatLog.MsgData.SenderNickname == "" { + sendIDs = append(sendIDs, chatLog.MsgData.SendID) } - switch chatLog.SessionType { + switch chatLog.MsgData.SessionType { case constant.SingleChatType, constant.NotificationChatType: - recvIDs = append(recvIDs, chatLog.RecvID) + recvIDs = append(recvIDs, chatLog.MsgData.RecvID) case constant.WriteGroupChatType, constant.ReadGroupChatType: - groupIDs = append(groupIDs, chatLog.GroupID) + groupIDs = append(groupIDs, chatLog.MsgData.GroupID) } } + // Retrieve sender and receiver information if len(sendIDs) != 0 { sendInfos, err := m.UserLocalCache.GetUsersInfo(ctx, sendIDs) @@ -146,6 +184,7 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq sendMap[sendInfo.UserID] = sendInfo.Nickname } } + if len(recvIDs) != 0 { recvInfos, err := m.UserLocalCache.GetUsersInfo(ctx, recvIDs) if err != nil { @@ -171,20 +210,21 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq } } } + // Construct response with updated information for _, chatLog := range chatLogs { pbchatLog := &msg.ChatLog{} - datautil.CopyStructFields(pbchatLog, chatLog) - pbchatLog.SendTime = chatLog.SendTime - pbchatLog.CreateTime = chatLog.CreateTime - if chatLog.SenderNickname == "" { - pbchatLog.SenderNickname = sendMap[chatLog.SendID] + datautil.CopyStructFields(pbchatLog, chatLog.MsgData) + pbchatLog.SendTime = chatLog.MsgData.SendTime + pbchatLog.CreateTime = chatLog.MsgData.CreateTime + if chatLog.MsgData.SenderNickname == "" { + pbchatLog.SenderNickname = sendMap[chatLog.MsgData.SendID] } - switch chatLog.SessionType { + switch chatLog.MsgData.SessionType { case constant.SingleChatType, constant.NotificationChatType: - pbchatLog.RecvNickname = recvMap[chatLog.RecvID] - case constant.WriteGroupChatType, constant.ReadGroupChatType: - groupInfo := groupMap[chatLog.GroupID] + pbchatLog.RecvNickname = recvMap[chatLog.MsgData.RecvID] + case constant.ReadGroupChatType: + groupInfo := groupMap[chatLog.MsgData.GroupID] pbchatLog.SenderFaceURL = groupInfo.FaceURL pbchatLog.GroupMemberCount = groupInfo.MemberCount // Reflects actual member count pbchatLog.RecvID = groupInfo.GroupID @@ -192,7 +232,9 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq pbchatLog.GroupOwner = groupInfo.OwnerUserID pbchatLog.GroupType = groupInfo.GroupType } - resp.ChatLogs = append(resp.ChatLogs, pbchatLog) + searchChatLog := &msg.SearchChatLog{ChatLog: pbchatLog, IsRevoked: chatLog.IsRevoked} + + resp.ChatLogs = append(resp.ChatLogs, searchChatLog) } resp.ChatLogsNum = int32(total) return resp, nil diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index 6b52181b6c..9130589321 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -312,16 +312,20 @@ func (s *friendServer) GetPaginationFriendsApplyTo(ctx context.Context, req *rel if err := s.userRpcClient.Access(ctx, req.UserID); err != nil { return nil, err } + total, friendRequests, err := s.db.PageFriendRequestToMe(ctx, req.UserID, req.Pagination) if err != nil { return nil, err } + resp = &relation.GetPaginationFriendsApplyToResp{} resp.FriendRequests, err = convert.FriendRequestDB2Pb(ctx, friendRequests, s.userRpcClient.GetUsersInfoMap) if err != nil { return nil, err } + resp.Total = int32(total) + return resp, nil } diff --git a/internal/rpc/third/log.go b/internal/rpc/third/log.go index 68d7088b0c..657ea16893 100644 --- a/internal/rpc/third/log.go +++ b/internal/rpc/third/log.go @@ -17,9 +17,10 @@ package third import ( "context" "crypto/rand" - relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + relationtb "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/constant" diff --git a/internal/rpc/user/callback.go b/internal/rpc/user/callback.go index 1bdf399d22..b58053c5ac 100644 --- a/internal/rpc/user/callback.go +++ b/internal/rpc/user/callback.go @@ -16,6 +16,7 @@ package user import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/webhook" "github.com/openimsdk/tools/utils/datautil" @@ -88,7 +89,6 @@ func (s *userServer) webhookBeforeUserRegister(ctx context.Context, before *conf return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeUserRegisterReq{ CallbackCommand: cbapi.CallbackBeforeUserRegisterCommand, - Secret: req.Secret, Users: req.Users, } @@ -108,7 +108,6 @@ func (s *userServer) webhookBeforeUserRegister(ctx context.Context, before *conf func (s *userServer) webhookAfterUserRegister(ctx context.Context, after *config.AfterConfig, req *pbuser.UserRegisterReq) { cbReq := &cbapi.CallbackAfterUserRegisterReq{ CallbackCommand: cbapi.CallbackAfterUserRegisterCommand, - Secret: req.Secret, Users: req.Users, } diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go index 99b272006f..4e7823306f 100644 --- a/internal/rpc/user/online.go +++ b/internal/rpc/user/online.go @@ -2,6 +2,8 @@ package user import ( "context" + "github.com/openimsdk/tools/utils/datautil" + "github.com/openimsdk/protocol/constant" pbuser "github.com/openimsdk/protocol/user" ) @@ -80,3 +82,22 @@ func (s *userServer) SetUserOnlineStatus(ctx context.Context, req *pbuser.SetUse } return &pbuser.SetUserOnlineStatusResp{}, nil } + +func (s *userServer) GetAllOnlineUsers(ctx context.Context, req *pbuser.GetAllOnlineUsersReq) (*pbuser.GetAllOnlineUsersResp, error) { + resMap, nextCursor, err := s.online.GetAllOnlineUsers(ctx, req.Cursor) + if err != nil { + return nil, err + } + resp := &pbuser.GetAllOnlineUsersResp{ + StatusList: make([]*pbuser.OnlineStatus, 0, len(resMap)), + NextCursor: nextCursor, + } + for userID, plats := range resMap { + resp.StatusList = append(resp.StatusList, &pbuser.OnlineStatus{ + UserID: userID, + Status: int32(datautil.If(len(plats) > 0, constant.Online, constant.Offline)), + PlatformIDs: plats, + }) + } + return resp, nil +} diff --git a/internal/rpc/user/user.go b/internal/rpc/user/user.go index 8b22c8f9b7..4669ed513f 100644 --- a/internal/rpc/user/user.go +++ b/internal/rpc/user/user.go @@ -47,7 +47,6 @@ import ( "github.com/openimsdk/tools/db/pagination" registry "github.com/openimsdk/tools/discovery" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc" ) @@ -263,10 +262,11 @@ func (s *userServer) UserRegister(ctx context.Context, req *pbuser.UserRegisterR if len(req.Users) == 0 { return nil, errs.ErrArgs.WrapMsg("users is empty") } - if req.Secret != s.config.Share.Secret { - log.ZDebug(ctx, "UserRegister", s.config.Share.Secret, req.Secret) - return nil, errs.ErrNoPermission.WrapMsg("secret invalid") + + if err = authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { + return nil, err } + if datautil.DuplicateAny(req.Users, func(e *sdkws.UserInfo) string { return e.UserID }) { return nil, errs.ErrArgs.WrapMsg("userID repeated") } diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 337272d69d..dbb4e34f61 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -79,13 +79,13 @@ func Start(ctx context.Context, config *CronTaskConfig) error { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZInfo(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + log.ZDebug(ctx, "clear chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) if _, err := msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Timestamp: deltime.UnixMilli()}); err != nil { log.ZError(ctx, "cron clear chat records failed", err, "deltime", deltime, "cont", time.Since(now)) return } - log.ZInfo(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) + log.ZDebug(ctx, "cron clear chat records success", "deltime", deltime, "cont", time.Since(now)) } if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { return errs.Wrap(err) @@ -95,7 +95,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { msgDestructFunc := func() { now := time.Now() ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZInfo(ctx, "msg destruct cron start", "now", now) + log.ZDebug(ctx, "msg destruct cron start", "now", now) conversations, err := conversationClient.GetConversationsNeedDestructMsgs(ctx, &pbconversation.GetConversationsNeedDestructMsgsReq{}) if err != nil { @@ -108,7 +108,7 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return } } - log.ZInfo(ctx, "msg destruct cron task completed", "cont", time.Since(now)) + log.ZDebug(ctx, "msg destruct cron task completed", "cont", time.Since(now)) } if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, msgDestructFunc); err != nil { return errs.Wrap(err) @@ -119,18 +119,18 @@ func Start(ctx context.Context, config *CronTaskConfig) error { // now := time.Now() // deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) // ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - // log.ZInfo(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + // log.ZDebug(ctx, "deleteoutDatedData ", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) // if _, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli()}); err != nil { // log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) // return // } - // log.ZInfo(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + // log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) // } // if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { // return errs.Wrap(err) // } - log.ZInfo(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) + log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) crontab.Start() <-ctx.Done() return nil diff --git a/pkg/callbackstruct/constant.go b/pkg/callbackstruct/constant.go index 89062ee0a0..5d136d3da3 100644 --- a/pkg/callbackstruct/constant.go +++ b/pkg/callbackstruct/constant.go @@ -18,9 +18,9 @@ const ( CallbackBeforeInviteJoinGroupCommand = "callbackBeforeInviteJoinGroupCommand" CallbackAfterJoinGroupCommand = "callbackAfterJoinGroupCommand" CallbackAfterSetGroupInfoCommand = "callbackAfterSetGroupInfoCommand" - CallbackAfterSetGroupInfoEXCommand = "callbackAfterSetGroupInfoCommandEX" + CallbackAfterSetGroupInfoExCommand = "callbackAfterSetGroupInfoExCommand" CallbackBeforeSetGroupInfoCommand = "callbackBeforeSetGroupInfoCommand" - CallbackBeforeSetGroupInfoEXCommand = "callbackBeforeSetGroupInfoEXCommand" + CallbackBeforeSetGroupInfoExCommand = "callbackBeforeSetGroupInfoExCommand" CallbackAfterRevokeMsgCommand = "callbackBeforeAfterMsgCommand" CallbackBeforeAddBlackCommand = "callbackBeforeAddBlackCommand" CallbackAfterAddFriendCommand = "callbackAfterAddFriendCommand" diff --git a/pkg/callbackstruct/group.go b/pkg/callbackstruct/group.go index 7fefa5b926..1f9b848cf9 100644 --- a/pkg/callbackstruct/group.go +++ b/pkg/callbackstruct/group.go @@ -244,11 +244,11 @@ type CallbackAfterSetGroupInfoResp struct { CommonCallbackResp } -type CallbackBeforeSetGroupInfoEXReq struct { +type CallbackBeforeSetGroupInfoExReq struct { CallbackCommand `json:"callbackCommand"` OperationID string `json:"operationID"` GroupID string `json:"groupID"` - GroupName string `json:"groupName"` + GroupName *wrapperspb.StringValue `json:"groupName"` Notification *wrapperspb.StringValue `json:"notification"` Introduction *wrapperspb.StringValue `json:"introduction"` FaceURL *wrapperspb.StringValue `json:"faceURL"` @@ -258,10 +258,10 @@ type CallbackBeforeSetGroupInfoEXReq struct { ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` } -type CallbackBeforeSetGroupInfoEXResp struct { +type CallbackBeforeSetGroupInfoExResp struct { CommonCallbackResp GroupID string `json:"groupID"` - GroupName string `json:"groupName"` + GroupName *wrapperspb.StringValue `json:"groupName"` Notification *wrapperspb.StringValue `json:"notification"` Introduction *wrapperspb.StringValue `json:"introduction"` FaceURL *wrapperspb.StringValue `json:"faceURL"` @@ -271,11 +271,11 @@ type CallbackBeforeSetGroupInfoEXResp struct { ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` } -type CallbackAfterSetGroupInfoEXReq struct { +type CallbackAfterSetGroupInfoExReq struct { CallbackCommand `json:"callbackCommand"` OperationID string `json:"operationID"` GroupID string `json:"groupID"` - GroupName string `json:"groupName"` + GroupName *wrapperspb.StringValue `json:"groupName"` Notification *wrapperspb.StringValue `json:"notification"` Introduction *wrapperspb.StringValue `json:"introduction"` FaceURL *wrapperspb.StringValue `json:"faceURL"` @@ -285,6 +285,6 @@ type CallbackAfterSetGroupInfoEXReq struct { ApplyMemberFriend *wrapperspb.Int32Value `json:"applyMemberFriend"` } -type CallbackAfterSetGroupInfoEXResp struct { +type CallbackAfterSetGroupInfoExResp struct { CommonCallbackResp } diff --git a/pkg/callbackstruct/user.go b/pkg/callbackstruct/user.go index 504c7ffb7e..d8640da29a 100644 --- a/pkg/callbackstruct/user.go +++ b/pkg/callbackstruct/user.go @@ -72,7 +72,6 @@ type CallbackAfterUpdateUserInfoExResp struct { type CallbackBeforeUserRegisterReq struct { CallbackCommand `json:"callbackCommand"` - Secret string `json:"secret"` Users []*sdkws.UserInfo `json:"users"` } @@ -83,7 +82,6 @@ type CallbackBeforeUserRegisterResp struct { type CallbackAfterUserRegisterReq struct { CallbackCommand `json:"callbackCommand"` - Secret string `json:"secret"` Users []*sdkws.UserInfo `json:"users"` } diff --git a/pkg/common/cmd/push.go b/pkg/common/cmd/push.go index c9b8b1c245..ca22a697d2 100644 --- a/pkg/common/cmd/push.go +++ b/pkg/common/cmd/push.go @@ -37,7 +37,6 @@ func NewPushRpcCmd() *PushRpcCmd { ret.configMap = map[string]any{ OpenIMPushCfgFileName: &pushConfig.RpcConfig, RedisConfigFileName: &pushConfig.RedisConfig, - MongodbConfigFileName: &pushConfig.MongodbConfig, KafkaConfigFileName: &pushConfig.KafkaConfig, ShareFileName: &pushConfig.Share, NotificationFileName: &pushConfig.NotificationConfig, diff --git a/pkg/common/cmd/root.go b/pkg/common/cmd/root.go index b43f86557f..5edea43773 100644 --- a/pkg/common/cmd/root.go +++ b/pkg/common/cmd/root.go @@ -129,10 +129,11 @@ func (r *RootCmd) applyOptions(opts ...func(*CmdOpts)) *CmdOpts { } func (r *RootCmd) initializeLogger(cmdOpts *CmdOpts) error { - err := log.InitFromConfig( + err := log.InitLoggerFromConfig( cmdOpts.loggerPrefixName, r.processName, + "", "", r.log.RemainLogLevel, r.log.IsStdout, r.log.IsJson, diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 5261e034ce..da0d6f1a12 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -73,18 +73,21 @@ type Mongo struct { MaxRetry int `mapstructure:"maxRetry"` } type Kafka struct { - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - ProducerAck string `mapstructure:"producerAck"` - CompressType string `mapstructure:"compressType"` - Address []string `mapstructure:"address"` - ToRedisTopic string `mapstructure:"toRedisTopic"` - ToMongoTopic string `mapstructure:"toMongoTopic"` - ToPushTopic string `mapstructure:"toPushTopic"` - ToRedisGroupID string `mapstructure:"toRedisGroupID"` - ToMongoGroupID string `mapstructure:"toMongoGroupID"` - ToPushGroupID string `mapstructure:"toPushGroupID"` - Tls TLSConfig `mapstructure:"tls"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + ProducerAck string `mapstructure:"producerAck"` + CompressType string `mapstructure:"compressType"` + Address []string `mapstructure:"address"` + ToRedisTopic string `mapstructure:"toRedisTopic"` + ToMongoTopic string `mapstructure:"toMongoTopic"` + ToPushTopic string `mapstructure:"toPushTopic"` + ToOfflinePushTopic string `mapstructure:"toOfflinePushTopic"` + ToRedisGroupID string `mapstructure:"toRedisGroupID"` + ToMongoGroupID string `mapstructure:"toMongoGroupID"` + ToPushGroupID string `mapstructure:"toPushGroupID"` + ToOfflineGroupID string `mapstructure:"toOfflinePushGroupID"` + + Tls TLSConfig `mapstructure:"tls"` } type TLSConfig struct { EnableTLS bool `mapstructure:"enableTLS"` @@ -97,8 +100,9 @@ type TLSConfig struct { type API struct { Api struct { - ListenIP string `mapstructure:"listenIP"` - Ports []int `mapstructure:"ports"` + ListenIP string `mapstructure:"listenIP"` + Ports []int `mapstructure:"ports"` + CompressionLevel int `mapstructure:"compressionLevel"` } `mapstructure:"api"` Prometheus struct { Enable bool `mapstructure:"enable"` @@ -220,6 +224,7 @@ type Push struct { BadgeCount bool `mapstructure:"badgeCount"` Production bool `mapstructure:"production"` } `mapstructure:"iosPush"` + FullUserCache bool `mapstructure:"fullUserCache"` } type Auth struct { @@ -336,7 +341,8 @@ type Redis struct { Password string `mapstructure:"password"` ClusterMode bool `mapstructure:"clusterMode"` DB int `mapstructure:"storage"` - MaxRetry int `mapstructure:"MaxRetry"` + MaxRetry int `mapstructure:"maxRetry"` + PoolSize int `mapstructure:"poolSize"` } type BeforeConfig struct { @@ -422,8 +428,8 @@ type Webhooks struct { BeforeInviteUserToGroup BeforeConfig `mapstructure:"beforeInviteUserToGroup"` AfterSetGroupInfo AfterConfig `mapstructure:"afterSetGroupInfo"` BeforeSetGroupInfo BeforeConfig `mapstructure:"beforeSetGroupInfo"` - AfterSetGroupInfoEX AfterConfig `mapstructure:"afterSetGroupInfoEX"` - BeforeSetGroupInfoEX BeforeConfig `mapstructure:"beforeSetGroupInfoEX"` + AfterSetGroupInfoEx AfterConfig `mapstructure:"afterSetGroupInfoEx"` + BeforeSetGroupInfoEx BeforeConfig `mapstructure:"beforeSetGroupInfoEx"` AfterRevokeMsg AfterConfig `mapstructure:"afterRevokeMsg"` BeforeAddBlack BeforeConfig `mapstructure:"beforeAddBlack"` AfterAddFriend AfterConfig `mapstructure:"afterAddFriend"` @@ -474,6 +480,7 @@ func (r *Redis) Build() *redisutil.Config { Password: r.Password, DB: r.DB, MaxRetry: r.MaxRetry, + PoolSize: r.PoolSize, } } diff --git a/pkg/common/prommetrics/grpc_push.go b/pkg/common/prommetrics/grpc_push.go index 0b6c3e76f3..5c966310f7 100644 --- a/pkg/common/prommetrics/grpc_push.go +++ b/pkg/common/prommetrics/grpc_push.go @@ -23,4 +23,8 @@ var ( Name: "msg_offline_push_failed_total", Help: "The number of msg failed offline pushed", }) + MsgLoneTimePushCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "msg_long_time_push_total", + Help: "The number of messages with a push time exceeding 10 seconds", + }) ) diff --git a/pkg/common/prommetrics/rpc.go b/pkg/common/prommetrics/rpc.go index dc16322dab..7162fa7e80 100644 --- a/pkg/common/prommetrics/rpc.go +++ b/pkg/common/prommetrics/rpc.go @@ -47,9 +47,17 @@ func GetGrpcCusMetrics(registerName string, share *config.Share) []prometheus.Co case share.RpcRegisterName.MessageGateway: return []prometheus.Collector{OnlineUserGauge} case share.RpcRegisterName.Msg: - return []prometheus.Collector{SingleChatMsgProcessSuccessCounter, SingleChatMsgProcessFailedCounter, GroupChatMsgProcessSuccessCounter, GroupChatMsgProcessFailedCounter} + return []prometheus.Collector{ + SingleChatMsgProcessSuccessCounter, + SingleChatMsgProcessFailedCounter, + GroupChatMsgProcessSuccessCounter, + GroupChatMsgProcessFailedCounter, + } case share.RpcRegisterName.Push: - return []prometheus.Collector{MsgOfflinePushFailedCounter} + return []prometheus.Collector{ + MsgOfflinePushFailedCounter, + MsgLoneTimePushCounter, + } case share.RpcRegisterName.Auth: return []prometheus.Collector{UserLoginCounter} case share.RpcRegisterName.User: diff --git a/pkg/common/startrpc/start.go b/pkg/common/startrpc/start.go index 4091a5f6e6..fb8782d304 100644 --- a/pkg/common/startrpc/start.go +++ b/pkg/common/startrpc/start.go @@ -25,7 +25,6 @@ import ( "os" "os/signal" "strconv" - "sync" "syscall" "time" @@ -35,7 +34,6 @@ import ( "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mw" - "github.com/openimsdk/tools/system/program" "github.com/openimsdk/tools/utils/network" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -54,6 +52,7 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo log.CInfo(ctx, "RPC server is initializing", "rpcRegisterName", rpcRegisterName, "rpcPort", rpcPort, "prometheusPorts", prometheusConfig.Ports) rpcTcpAddr := net.JoinHostPort(network.GetListenIP(listenIP), strconv.Itoa(rpcPort)) + listener, err := net.Listen( "tcp", rpcTcpAddr, @@ -61,7 +60,6 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo if err != nil { return errs.WrapMsg(err, "listen err", "rpcTcpAddr", rpcTcpAddr) } - defer listener.Close() client, err := kdisc.NewDiscoveryRegister(discovery, share) if err != nil { @@ -92,10 +90,6 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo } srv := grpc.NewServer(options...) - once := sync.Once{} - defer func() { - once.Do(srv.GracefulStop) - }() err = rpcFn(ctx, config, client, srv) if err != nil { @@ -113,9 +107,8 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo } var ( - netDone = make(chan struct{}, 2) - netErr error - httpServer *http.Server + netDone = make(chan struct{}, 2) + netErr error ) if prometheusConfig.Enable { go func() { @@ -152,18 +145,11 @@ func Start[T any](ctx context.Context, discovery *config.Discovery, prometheusCo signal.Notify(sigs, syscall.SIGTERM) select { case <-sigs: - program.SIGTERMExit() - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := gracefulStopWithCtx(ctx, srv.GracefulStop); err != nil { return err } - ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - err := httpServer.Shutdown(ctx) - if err != nil { - return errs.WrapMsg(err, "shutdown err") - } return nil case <-netDone: return netErr diff --git a/pkg/common/storage/cache/cachekey/online.go b/pkg/common/storage/cache/cachekey/online.go index 164e5f2f46..40f09cb5ae 100644 --- a/pkg/common/storage/cache/cachekey/online.go +++ b/pkg/common/storage/cache/cachekey/online.go @@ -1,6 +1,9 @@ package cachekey -import "time" +import ( + "strings" + "time" +) const ( OnlineKey = "ONLINE:" @@ -11,3 +14,7 @@ const ( func GetOnlineKey(userID string) string { return OnlineKey + userID } + +func GetOnlineKeyUserID(key string) string { + return strings.TrimPrefix(key, OnlineKey) +} diff --git a/pkg/common/storage/cache/online.go b/pkg/common/storage/cache/online.go index 7669c8a118..d21ae616a6 100644 --- a/pkg/common/storage/cache/online.go +++ b/pkg/common/storage/cache/online.go @@ -5,4 +5,5 @@ import "context" type OnlineCache interface { GetOnline(ctx context.Context, userID string) ([]int32, error) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error + GetAllOnlineUsers(ctx context.Context, cursor uint64) (map[string][]int32, uint64, error) } diff --git a/pkg/common/storage/cache/redis/batch.go b/pkg/common/storage/cache/redis/batch.go index 4d65c59298..1810ac9939 100644 --- a/pkg/common/storage/cache/redis/batch.go +++ b/pkg/common/storage/cache/redis/batch.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "github.com/dtm-labs/rockscache" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" "golang.org/x/sync/singleflight" @@ -65,6 +66,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac } bs, err := json.Marshal(value) if err != nil { + log.ZError(ctx, "marshal failed", err) return nil, err } cacheIndex[index] = string(bs) @@ -72,7 +74,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac return cacheIndex, nil }) if err != nil { - return nil, err + return nil, errs.WrapMsg(err, "FetchBatch2 failed") } for index, data := range indexCache { if data == "" { @@ -80,7 +82,7 @@ func batchGetCache2[K comparable, V any](ctx context.Context, rcClient *rockscac } var value V if err := json.Unmarshal([]byte(data), &value); err != nil { - return nil, err + return nil, errs.WrapMsg(err, "Unmarshal failed") } if cb, ok := any(&value).(BatchCacheCallback[K]); ok { cb.BatchCache(keyId[keys[index]]) diff --git a/pkg/common/storage/cache/redis/batch_handler.go b/pkg/common/storage/cache/redis/batch_handler.go index f9923e198e..420ebdf777 100644 --- a/pkg/common/storage/cache/redis/batch_handler.go +++ b/pkg/common/storage/cache/redis/batch_handler.go @@ -28,6 +28,10 @@ import ( "time" ) +const ( + rocksCacheTimeout = 11 * time.Second +) + // BatchDeleterRedis is a concrete implementation of the BatchDeleter interface based on Redis and RocksCache. type BatchDeleterRedis struct { redisClient redis.UniversalClient @@ -106,6 +110,8 @@ func (c *BatchDeleterRedis) AddKeys(keys ...string) { // GetRocksCacheOptions returns the default configuration options for RocksCache. func GetRocksCacheOptions() *rockscache.Options { opts := rockscache.NewDefaultOptions() + opts.LockExpire = rocksCacheTimeout + opts.WaitReplicasTimeout = rocksCacheTimeout opts.StrongConsistency = true opts.RandomExpireAdjustment = 0.2 @@ -118,7 +124,7 @@ func getCache[T any](ctx context.Context, rcClient *rockscache.Client, key strin v, err := rcClient.Fetch2(ctx, key, expire, func() (s string, err error) { t, err = fn(ctx) if err != nil { - log.ZError(ctx, "getCache query database failed", err, "key", key) + //log.ZError(ctx, "getCache query database failed", err, "key", key) return "", err } bs, err := json.Marshal(t) diff --git a/pkg/common/storage/cache/redis/online.go b/pkg/common/storage/cache/redis/online.go index ee1db7e23b..b6c90264e1 100644 --- a/pkg/common/storage/cache/redis/online.go +++ b/pkg/common/storage/cache/redis/online.go @@ -2,8 +2,10 @@ package redis import ( "context" + "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" + "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" @@ -49,6 +51,36 @@ func (s *userOnline) GetOnline(ctx context.Context, userID string) ([]int32, err return platformIDs, nil } +func (s *userOnline) GetAllOnlineUsers(ctx context.Context, cursor uint64) (map[string][]int32, uint64, error) { + result := make(map[string][]int32) + + keys, nextCursor, err := s.rdb.Scan(ctx, cursor, fmt.Sprintf("%s*", cachekey.OnlineKey), constant.ParamMaxLength).Result() + if err != nil { + return nil, 0, err + } + + for _, key := range keys { + userID := cachekey.GetOnlineKeyUserID(key) + strValues, err := s.rdb.ZRange(ctx, key, 0, -1).Result() + if err != nil { + return nil, 0, err + } + + values := make([]int32, 0, len(strValues)) + for _, value := range strValues { + intValue, err := strconv.Atoi(value) + if err != nil { + return nil, 0, errs.Wrap(err) + } + values = append(values, int32(intValue)) + } + + result[userID] = values + } + + return result, nextCursor, nil +} + func (s *userOnline) SetUserOnline(ctx context.Context, userID string, online, offline []int32) error { script := ` local key = KEYS[1] diff --git a/pkg/common/storage/cache/redis/token.go b/pkg/common/storage/cache/redis/token.go index b822596582..24e9c30050 100644 --- a/pkg/common/storage/cache/redis/token.go +++ b/pkg/common/storage/cache/redis/token.go @@ -19,8 +19,8 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/stringutil" "github.com/redis/go-redis/v9" + "strconv" "time" ) @@ -58,9 +58,12 @@ func (c *tokenCache) GetTokensWithoutError(ctx context.Context, userID string, p } mm := make(map[string]int) for k, v := range m { - mm[k] = stringutil.StringToInt(v) + state, err := strconv.Atoi(v) + if err != nil { + return nil, errs.WrapMsg(err, "redis token value is not int", "value", v, "userID", userID, "platformID", platformID) + } + mm[k] = state } - return mm, nil } diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 7f884165de..16a7b1c9b7 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -26,7 +26,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/convert" - "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" "github.com/openimsdk/protocol/constant" pbmsg "github.com/openimsdk/protocol/msg" @@ -47,16 +46,10 @@ const ( // CommonMsgDatabase defines the interface for message database operations. type CommonMsgDatabase interface { - // BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation. - BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error // RevokeMsg revokes a message in a conversation. RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error // MarkSingleChatMsgsAsRead marks messages as read for a single chat by sequence numbers. MarkSingleChatMsgsAsRead(ctx context.Context, userID string, conversationID string, seqs []int64) error - // DeleteMessagesFromCache deletes message caches from Redis by sequence numbers. - DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error - // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. - BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) // GetMsgBySeqsRange retrieves messages from MongoDB by a range of sequence numbers. GetMsgBySeqsRange(ctx context.Context, userID string, conversationID string, begin, end, num, userMaxSeq int64) (minSeq int64, maxSeq int64, seqMsg []*sdkws.MsgData, err error) // GetMsgBySeqs retrieves messages for large groups from MongoDB by sequence numbers. @@ -77,7 +70,6 @@ type CommonMsgDatabase interface { SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error - SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) GetHasReadSeq(ctx context.Context, userID string, conversationID string) (int64, error) UserSetHasReadSeqs(ctx context.Context, userID string, hasReadSeqs map[string]int64) error @@ -86,13 +78,11 @@ type CommonMsgDatabase interface { //GetConversationMinMaxSeqInMongoAndCache(ctx context.Context, conversationID string) (minSeqMongo, maxSeqMongo, minSeqCache, maxSeqCache int64, err error) SetSendMsgStatus(ctx context.Context, id string, status int32) error GetSendMsgStatus(ctx context.Context, id string) (int32, error) - SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) + SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*pbmsg.SearchedMsgData, err error) FindOneByDocIDs(ctx context.Context, docIDs []string, seqs map[string]int64) (map[string]*sdkws.MsgData, error) // to mq MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error - MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) - MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error RangeUserSendCount(ctx context.Context, start time.Time, end time.Time, group bool, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, users []*model.UserCount, dateCount map[string]int64, err error) RangeGroupSendCount(ctx context.Context, start time.Time, end time.Time, ase bool, pageNumber int32, showNumber int32) (msgCount int64, userCount int64, groups []*model.GroupCount, dateCount map[string]int64, err error) @@ -114,22 +104,12 @@ func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser if err != nil { return nil, err } - producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) - if err != nil { - return nil, err - } - producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic) - if err != nil { - return nil, err - } return &commonMsgDatabase{ msgDocDatabase: msgDocModel, msg: msg, seqUser: seqUser, seqConversation: seqConversation, producer: producerToRedis, - producerToMongo: producerToMongo, - producerToPush: producerToPush, }, nil } @@ -140,8 +120,6 @@ type commonMsgDatabase struct { seqConversation cache.SeqConversationCache seqUser cache.SeqUser producer *kafka.Producer - producerToMongo *kafka.Producer - producerToPush *kafka.Producer } func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sdkws.MsgData) error { @@ -149,23 +127,6 @@ func (db *commonMsgDatabase) MsgToMQ(ctx context.Context, key string, msg2mq *sd return err } -func (db *commonMsgDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { - partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) - if err != nil { - log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq) - return 0, 0, err - } - return partition, offset, nil -} - -func (db *commonMsgDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error { - if len(messages) > 0 { - _, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) - return err - } - return nil -} - func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { if len(fields) == 0 { return nil @@ -267,52 +228,6 @@ func (db *commonMsgDatabase) BatchInsertBlock(ctx context.Context, conversationI return nil } -func (db *commonMsgDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error { - if len(msgList) == 0 { - return errs.ErrArgs.WrapMsg("msgList is empty") - } - msgs := make([]any, len(msgList)) - for i, msg := range msgList { - if msg == nil { - continue - } - var offlinePushModel *model.OfflinePushModel - if msg.OfflinePushInfo != nil { - offlinePushModel = &model.OfflinePushModel{ - Title: msg.OfflinePushInfo.Title, - Desc: msg.OfflinePushInfo.Desc, - Ex: msg.OfflinePushInfo.Ex, - IOSPushSound: msg.OfflinePushInfo.IOSPushSound, - IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, - } - } - msgs[i] = &model.MsgDataModel{ - SendID: msg.SendID, - RecvID: msg.RecvID, - GroupID: msg.GroupID, - ClientMsgID: msg.ClientMsgID, - ServerMsgID: msg.ServerMsgID, - SenderPlatformID: msg.SenderPlatformID, - SenderNickname: msg.SenderNickname, - SenderFaceURL: msg.SenderFaceURL, - SessionType: msg.SessionType, - MsgFrom: msg.MsgFrom, - ContentType: msg.ContentType, - Content: string(msg.Content), - Seq: msg.Seq, - SendTime: msg.SendTime, - CreateTime: msg.CreateTime, - Status: msg.Status, - Options: msg.Options, - OfflinePush: offlinePushModel, - AtUserIDList: msg.AtUserIDList, - AttachedInfo: msg.AttachedInfo, - Ex: msg.Ex, - } - } - return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) -} - func (db *commonMsgDatabase) RevokeMsg(ctx context.Context, conversationID string, seq int64, revoke *model.RevokeModel) error { return db.BatchInsertBlock(ctx, conversationID, []any{revoke}, updateKeyRevoke, seq) } @@ -332,56 +247,6 @@ func (db *commonMsgDatabase) MarkSingleChatMsgsAsRead(ctx context.Context, userI return nil } -func (db *commonMsgDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { - return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) -} - -func (db *commonMsgDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { - for userID, seq := range userSeqMap { - if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { - return err - } - } - return nil -} - -func (db *commonMsgDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { - lenList := len(msgs) - if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { - return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() - } - if lenList < 1 { - return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() - } - currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) - if err != nil { - log.ZError(ctx, "storage.seq.Malloc", err) - return 0, false, err - } - isNew = currentMaxSeq == 0 - lastMaxSeq := currentMaxSeq - userSeqMap := make(map[string]int64) - for _, m := range msgs { - currentMaxSeq++ - m.Seq = currentMaxSeq - userSeqMap[m.SendID] = m.Seq - } - - failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) - if err != nil { - prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) - log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) - } else { - prommetrics.MsgInsertRedisSuccessCounter.Inc() - } - err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) - if err != nil { - log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - return lastMaxSeq, isNew, errs.Wrap(err) -} - func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversationID string, seqs []int64) (totalMsgs []*sdkws.MsgData, err error) { for docID, seqs := range db.msgTable.GetDocIDSeqsMap(conversationID, seqs) { // log.ZDebug(ctx, "getMsgBySeqs", "docID", docID, "seqs", seqs) @@ -581,7 +446,7 @@ func (db *commonMsgDatabase) GetMsgBySeqsRange(ctx context.Context, userID strin func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, conversationID string, seqs []int64) (int64, int64, []*sdkws.MsgData, error) { userMinSeq, err := db.seqUser.GetUserMinSeq(ctx, conversationID, userID) - if err != nil && errs.Unwrap(err) != redis.Nil { + if err != nil { return 0, 0, nil, err } minSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) @@ -592,15 +457,28 @@ func (db *commonMsgDatabase) GetMsgBySeqs(ctx context.Context, userID string, co if err != nil { return 0, 0, nil, err } - if userMinSeq < minSeq { + userMaxSeq, err := db.seqUser.GetUserMaxSeq(ctx, conversationID, userID) + if err != nil { + return 0, 0, nil, err + } + if userMinSeq > minSeq { minSeq = userMinSeq } - var newSeqs []int64 + if userMaxSeq > 0 && userMaxSeq < maxSeq { + maxSeq = userMaxSeq + } + newSeqs := make([]int64, 0, len(seqs)) for _, seq := range seqs { + if seq <= 0 { + continue + } if seq >= minSeq && seq <= maxSeq { newSeqs = append(newSeqs, seq) } } + if len(newSeqs) == 0 { + return minSeq, maxSeq, nil, nil + } successMsgs, failedSeqs, err := db.msg.GetMessagesBySeq(ctx, conversationID, newSeqs) if err != nil { if err != redis.Nil { @@ -809,10 +687,6 @@ func (db *commonMsgDatabase) SetHasReadSeq(ctx context.Context, userID string, c return db.seqUser.SetUserReadSeq(ctx, conversationID, userID, hasReadSeq) } -func (db *commonMsgDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) -} - func (db *commonMsgDatabase) GetHasReadSeqs(ctx context.Context, userID string, conversationIDs []string) (map[string]int64, error) { return db.seqUser.GetUserReadSeqs(ctx, userID, conversationIDs) } @@ -886,8 +760,8 @@ func (db *commonMsgDatabase) RangeGroupSendCount( return db.msgDocDatabase.RangeGroupSendCount(ctx, start, end, ase, pageNumber, showNumber) } -func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*sdkws.MsgData, err error) { - var totalMsgs []*sdkws.MsgData +func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.SearchMessageReq) (total int64, msgData []*pbmsg.SearchedMsgData, err error) { + var totalMsgs []*pbmsg.SearchedMsgData total, msgs, err := db.msgDocDatabase.SearchMessage(ctx, req) if err != nil { return 0, nil, err @@ -896,7 +770,13 @@ func (db *commonMsgDatabase) SearchMessage(ctx context.Context, req *pbmsg.Searc if msg.IsRead { msg.Msg.IsRead = true } - totalMsgs = append(totalMsgs, convert.MsgDB2Pb(msg.Msg)) + searchedMsgData := &pbmsg.SearchedMsgData{MsgData: convert.MsgDB2Pb(msg.Msg)} + + if msg.Revoke != nil { + searchedMsgData.IsRevoked = true + } + + totalMsgs = append(totalMsgs, searchedMsgData) } return total, totalMsgs, nil } diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go new file mode 100644 index 0000000000..5e540a2c33 --- /dev/null +++ b/pkg/common/storage/controller/msg_transfer.go @@ -0,0 +1,286 @@ +package controller + +import ( + "context" + + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + pbmsg "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" + "go.mongodb.org/mongo-driver/mongo" +) + +type MsgTransferDatabase interface { + // BatchInsertChat2DB inserts a batch of messages into the database for a specific conversation. + BatchInsertChat2DB(ctx context.Context, conversationID string, msgs []*sdkws.MsgData, currentMaxSeq int64) error + // DeleteMessagesFromCache deletes message caches from Redis by sequence numbers. + DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error + + // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. + BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) + SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error + + // to mq + MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) + MsgToMongoMQ(ctx context.Context, key, conversationID string, msgs []*sdkws.MsgData, lastSeq int64) error +} + +func NewMsgTransferDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (MsgTransferDatabase, error) { + conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) + if err != nil { + return nil, err + } + producerToMongo, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToMongoTopic) + if err != nil { + return nil, err + } + producerToPush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToPushTopic) + if err != nil { + return nil, err + } + return &msgTransferDatabase{ + msgDocDatabase: msgDocModel, + msg: msg, + seqUser: seqUser, + seqConversation: seqConversation, + producerToMongo: producerToMongo, + producerToPush: producerToPush, + }, nil +} + +type msgTransferDatabase struct { + msgDocDatabase database.Msg + msgTable model.MsgDocModel + msg cache.MsgCache + seqConversation cache.SeqConversationCache + seqUser cache.SeqUser + producerToMongo *kafka.Producer + producerToPush *kafka.Producer +} + +func (db *msgTransferDatabase) BatchInsertChat2DB(ctx context.Context, conversationID string, msgList []*sdkws.MsgData, currentMaxSeq int64) error { + if len(msgList) == 0 { + return errs.ErrArgs.WrapMsg("msgList is empty") + } + msgs := make([]any, len(msgList)) + for i, msg := range msgList { + if msg == nil { + continue + } + var offlinePushModel *model.OfflinePushModel + if msg.OfflinePushInfo != nil { + offlinePushModel = &model.OfflinePushModel{ + Title: msg.OfflinePushInfo.Title, + Desc: msg.OfflinePushInfo.Desc, + Ex: msg.OfflinePushInfo.Ex, + IOSPushSound: msg.OfflinePushInfo.IOSPushSound, + IOSBadgeCount: msg.OfflinePushInfo.IOSBadgeCount, + } + } + msgs[i] = &model.MsgDataModel{ + SendID: msg.SendID, + RecvID: msg.RecvID, + GroupID: msg.GroupID, + ClientMsgID: msg.ClientMsgID, + ServerMsgID: msg.ServerMsgID, + SenderPlatformID: msg.SenderPlatformID, + SenderNickname: msg.SenderNickname, + SenderFaceURL: msg.SenderFaceURL, + SessionType: msg.SessionType, + MsgFrom: msg.MsgFrom, + ContentType: msg.ContentType, + Content: string(msg.Content), + Seq: msg.Seq, + SendTime: msg.SendTime, + CreateTime: msg.CreateTime, + Status: msg.Status, + Options: msg.Options, + OfflinePush: offlinePushModel, + AtUserIDList: msg.AtUserIDList, + AttachedInfo: msg.AttachedInfo, + Ex: msg.Ex, + } + } + return db.BatchInsertBlock(ctx, conversationID, msgs, updateKeyMsg, msgList[0].Seq) +} + +func (db *msgTransferDatabase) BatchInsertBlock(ctx context.Context, conversationID string, fields []any, key int8, firstSeq int64) error { + if len(fields) == 0 { + return nil + } + num := db.msgTable.GetSingleGocMsgNum() + // num = 100 + for i, field := range fields { // Check the type of the field + var ok bool + switch key { + case updateKeyMsg: + var msg *model.MsgDataModel + msg, ok = field.(*model.MsgDataModel) + if msg != nil && msg.Seq != firstSeq+int64(i) { + return errs.ErrInternalServer.WrapMsg("seq is invalid") + } + case updateKeyRevoke: + _, ok = field.(*model.RevokeModel) + default: + return errs.ErrInternalServer.WrapMsg("key is invalid") + } + if !ok { + return errs.ErrInternalServer.WrapMsg("field type is invalid") + } + } + // Returns true if the document exists in the database, false if the document does not exist in the database + updateMsgModel := func(seq int64, i int) (bool, error) { + var ( + res *mongo.UpdateResult + err error + ) + docID := db.msgTable.GetDocID(conversationID, seq) + index := db.msgTable.GetMsgIndex(seq) + field := fields[i] + switch key { + case updateKeyMsg: + res, err = db.msgDocDatabase.UpdateMsg(ctx, docID, index, "msg", field) + case updateKeyRevoke: + res, err = db.msgDocDatabase.UpdateMsg(ctx, docID, index, "revoke", field) + } + if err != nil { + return false, err + } + return res.MatchedCount > 0, nil + } + tryUpdate := true + for i := 0; i < len(fields); i++ { + seq := firstSeq + int64(i) // Current sequence number + if tryUpdate { + matched, err := updateMsgModel(seq, i) + if err != nil { + return err + } + if matched { + continue // The current data has been updated, skip the current data + } + } + doc := model.MsgDocModel{ + DocID: db.msgTable.GetDocID(conversationID, seq), + Msg: make([]*model.MsgInfoModel, num), + } + var insert int // Inserted data number + for j := i; j < len(fields); j++ { + seq = firstSeq + int64(j) + if db.msgTable.GetDocID(conversationID, seq) != doc.DocID { + break + } + insert++ + switch key { + case updateKeyMsg: + doc.Msg[db.msgTable.GetMsgIndex(seq)] = &model.MsgInfoModel{ + Msg: fields[j].(*model.MsgDataModel), + } + case updateKeyRevoke: + doc.Msg[db.msgTable.GetMsgIndex(seq)] = &model.MsgInfoModel{ + Revoke: fields[j].(*model.RevokeModel), + } + } + } + for i, msgInfo := range doc.Msg { + if msgInfo == nil { + msgInfo = &model.MsgInfoModel{} + doc.Msg[i] = msgInfo + } + if msgInfo.DelList == nil { + doc.Msg[i].DelList = []string{} + } + } + if err := db.msgDocDatabase.Create(ctx, &doc); err != nil { + if mongo.IsDuplicateKeyError(err) { + i-- // already inserted + tryUpdate = true // next block use update mode + continue + } + return err + } + tryUpdate = false // The current block is inserted successfully, and the next block is inserted preferentially + i += insert - 1 // Skip the inserted data + } + return nil +} + +func (db *msgTransferDatabase) DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error { + return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) +} + +func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { + lenList := len(msgs) + if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { + return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() + } + if lenList < 1 { + return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() + } + currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) + if err != nil { + log.ZError(ctx, "storage.seq.Malloc", err) + return 0, false, err + } + isNew = currentMaxSeq == 0 + lastMaxSeq := currentMaxSeq + userSeqMap := make(map[string]int64) + for _, m := range msgs { + currentMaxSeq++ + m.Seq = currentMaxSeq + userSeqMap[m.SendID] = m.Seq + } + + failedNum, err := db.msg.SetMessagesToCache(ctx, conversationID, msgs) + if err != nil { + prommetrics.MsgInsertRedisFailedCounter.Add(float64(failedNum)) + log.ZError(ctx, "setMessageToCache error", err, "len", len(msgs), "conversationID", conversationID) + } else { + prommetrics.MsgInsertRedisSuccessCounter.Inc() + } + err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) + if err != nil { + log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) + prommetrics.SeqSetFailedCounter.Inc() + } + return lastMaxSeq, isNew, errs.Wrap(err) +} + +func (db *msgTransferDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil +} + +func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { + return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) +} + +func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { + partition, offset, err := db.producerToPush.SendMessage(ctx, key, &pbmsg.PushMsgDataToMQ{MsgData: msg2mq, ConversationID: conversationID}) + if err != nil { + log.ZError(ctx, "MsgToPushMQ", err, "key", key, "msg2mq", msg2mq) + return 0, 0, err + } + return partition, offset, nil +} + +func (db *msgTransferDatabase) MsgToMongoMQ(ctx context.Context, key, conversationID string, messages []*sdkws.MsgData, lastSeq int64) error { + if len(messages) > 0 { + _, _, err := db.producerToMongo.SendMessage(ctx, key, &pbmsg.MsgDataToMongoByMQ{LastSeq: lastSeq, ConversationID: conversationID, MsgData: messages}) + if err != nil { + log.ZError(ctx, "MsgToMongoMQ", err, "key", key, "conversationID", conversationID, "lastSeq", lastSeq) + return err + } + } + return nil +} diff --git a/pkg/common/storage/controller/push.go b/pkg/common/storage/controller/push.go index 199a0ba678..91ef126fe5 100644 --- a/pkg/common/storage/controller/push.go +++ b/pkg/common/storage/controller/push.go @@ -17,21 +17,45 @@ package controller import ( "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" + "github.com/openimsdk/protocol/push" + "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mq/kafka" ) type PushDatabase interface { DelFcmToken(ctx context.Context, userID string, platformID int) error + MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error } type pushDataBase struct { - cache cache.ThirdCache + cache cache.ThirdCache + producerToOfflinePush *kafka.Producer } -func NewPushDatabase(cache cache.ThirdCache) PushDatabase { - return &pushDataBase{cache: cache} +func NewPushDatabase(cache cache.ThirdCache, kafkaConf *config.Kafka) PushDatabase { + conf, err := kafka.BuildProducerConfig(*kafkaConf.Build()) + if err != nil { + return nil + } + producerToOfflinePush, err := kafka.NewKafkaProducer(conf, kafkaConf.Address, kafkaConf.ToOfflinePushTopic) + if err != nil { + return nil + } + return &pushDataBase{ + cache: cache, + producerToOfflinePush: producerToOfflinePush, + } } func (p *pushDataBase) DelFcmToken(ctx context.Context, userID string, platformID int) error { return p.cache.DelFcmToken(ctx, userID, platformID) } + +func (p *pushDataBase) MsgToOfflinePushMQ(ctx context.Context, key string, userIDs []string, msg2mq *sdkws.MsgData) error { + _, _, err := p.producerToOfflinePush.SendMessage(ctx, key, &push.PushMsgReq{MsgData: msg2mq, UserIDs: userIDs}) + log.ZInfo(ctx, "message is push to offlinePush topic", "key", key, "userIDs", userIDs, "msg", msg2mq.String()) + return err +} diff --git a/pkg/common/storage/controller/user.go b/pkg/common/storage/controller/user.go index 533eac78f1..3f34481a36 100644 --- a/pkg/common/storage/controller/user.go +++ b/pkg/common/storage/controller/user.go @@ -16,12 +16,13 @@ package controller import ( "context" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/db/tx" "github.com/openimsdk/tools/utils/datautil" - "time" "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/errs" @@ -111,10 +112,14 @@ func (u *userDatabase) InitOnce(ctx context.Context, users []*model.User) error // FindWithError Get the information of the specified user and return an error if the userID is not found. func (u *userDatabase) FindWithError(ctx context.Context, userIDs []string) (users []*model.User, err error) { userIDs = datautil.Distinct(userIDs) + + // TODO: Add logic to identify which user IDs are distinct and which user IDs were not found. + users, err = u.cache.GetUsersInfo(ctx, userIDs) if err != nil { return } + if len(users) != len(userIDs) { err = errs.ErrRecordNotFound.WrapMsg("userID not found") } diff --git a/pkg/localcache/lru/lru.go b/pkg/localcache/lru/lru.go index 2fedffc48b..726535c48c 100644 --- a/pkg/localcache/lru/lru.go +++ b/pkg/localcache/lru/lru.go @@ -20,7 +20,9 @@ type EvictCallback[K comparable, V any] simplelru.EvictCallback[K, V] type LRU[K comparable, V any] interface { Get(key K, fetch func() (V, error)) (V, error) + Set(key K, value V) SetHas(key K, value V) bool + GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) Del(key K) bool Stop() } diff --git a/pkg/localcache/lru/lru_expiration.go b/pkg/localcache/lru/lru_expiration.go index d27e670574..df6bacbf43 100644 --- a/pkg/localcache/lru/lru_expiration.go +++ b/pkg/localcache/lru/lru_expiration.go @@ -51,6 +51,11 @@ type ExpirationLRU[K comparable, V any] struct { target Target } +func (x *ExpirationLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + //TODO implement me + panic("implement me") +} + func (x *ExpirationLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { x.lock.Lock() v, ok := x.core.Get(key) @@ -99,5 +104,11 @@ func (x *ExpirationLRU[K, V]) SetHas(key K, value V) bool { return false } +func (x *ExpirationLRU[K, V]) Set(key K, value V) { + x.lock.Lock() + defer x.lock.Unlock() + x.core.Add(key, &expirationLruItem[V]{value: value}) +} + func (x *ExpirationLRU[K, V]) Stop() { } diff --git a/pkg/localcache/lru/lru_lazy.go b/pkg/localcache/lru/lru_lazy.go index e935c687c4..b4f0377a70 100644 --- a/pkg/localcache/lru/lru_lazy.go +++ b/pkg/localcache/lru/lru_lazy.go @@ -88,18 +88,75 @@ func (x *LayLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return v.value, v.err } -//func (x *LayLRU[K, V]) Set(key K, value V) { -// x.lock.Lock() -// x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) -// x.lock.Unlock() -//} -// +func (x *LayLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + var ( + err error + once sync.Once + ) + + res := make(map[K]V) + queries := make([]K, 0) + setVs := make(map[K]*layLruItem[V]) + for _, key := range keys { + x.lock.Lock() + v, ok := x.core.Get(key) + x.lock.Unlock() + if ok { + v.lock.Lock() + expires, value, err1 := v.expires, v.value, v.err + v.lock.Unlock() + if expires != 0 && expires > time.Now().UnixMilli() { + x.target.IncrGetHit() + res[key] = value + if err1 != nil { + once.Do(func() { + err = err1 + }) + } + continue + } + } + queries = append(queries, key) + } + values, err1 := fetch(queries) + if err1 != nil { + once.Do(func() { + err = err1 + }) + } + for key, val := range values { + v := &layLruItem[V]{} + v.value = val + + if err == nil { + v.expires = time.Now().Add(x.successTTL).UnixMilli() + x.target.IncrGetSuccess() + } else { + v.expires = time.Now().Add(x.failedTTL).UnixMilli() + x.target.IncrGetFailed() + } + setVs[key] = v + x.lock.Lock() + x.core.Add(key, v) + x.lock.Unlock() + res[key] = val + } + + return res, err +} + //func (x *LayLRU[K, V]) Has(key K) bool { // x.lock.Lock() // defer x.lock.Unlock() // return x.core.Contains(key) //} +func (x *LayLRU[K, V]) Set(key K, value V) { + x.lock.Lock() + defer x.lock.Unlock() + x.core.Add(key, &layLruItem[V]{value: value, expires: time.Now().Add(x.successTTL).UnixMilli()}) +} + func (x *LayLRU[K, V]) SetHas(key K, value V) bool { x.lock.Lock() defer x.lock.Unlock() diff --git a/pkg/localcache/lru/lru_slot.go b/pkg/localcache/lru/lru_slot.go index 4538ca20e4..077219b75f 100644 --- a/pkg/localcache/lru/lru_slot.go +++ b/pkg/localcache/lru/lru_slot.go @@ -32,6 +32,29 @@ type slotLRU[K comparable, V any] struct { hash func(k K) uint64 } +func (x *slotLRU[K, V]) GetBatch(keys []K, fetch func(keys []K) (map[K]V, error)) (map[K]V, error) { + var ( + slotKeys = make(map[uint64][]K) + vs = make(map[K]V) + ) + + for _, k := range keys { + index := x.getIndex(k) + slotKeys[index] = append(slotKeys[index], k) + } + + for k, v := range slotKeys { + batches, err := x.slots[k].GetBatch(v, fetch) + if err != nil { + return nil, err + } + for key, value := range batches { + vs[key] = value + } + } + return vs, nil +} + func (x *slotLRU[K, V]) getIndex(k K) uint64 { return x.hash(k) % x.n } @@ -40,6 +63,10 @@ func (x *slotLRU[K, V]) Get(key K, fetch func() (V, error)) (V, error) { return x.slots[x.getIndex(key)].Get(key, fetch) } +func (x *slotLRU[K, V]) Set(key K, value V) { + x.slots[x.getIndex(key)].Set(key, value) +} + func (x *slotLRU[K, V]) SetHas(key K, value V) bool { return x.slots[x.getIndex(key)].SetHas(key, value) } diff --git a/pkg/rpccache/conversation.go b/pkg/rpccache/conversation.go index 2a62c7bbd5..925d2a37ca 100644 --- a/pkg/rpccache/conversation.go +++ b/pkg/rpccache/conversation.go @@ -86,7 +86,7 @@ func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, co if err == nil { log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "userID", userID, "conversationID", conversationID, "value", val) } else { - log.ZError(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) + log.ZWarn(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID) } }() var cache cacheProto[pbconversation.Conversation] diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 2ffa1f1577..a02a0662d4 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -2,60 +2,197 @@ package rpccache import ( "context" + "fmt" + "github.com/openimsdk/protocol/constant" + "github.com/openimsdk/protocol/user" + "math/rand" + "strconv" + "sync" + "sync/atomic" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/open-im-server/v3/pkg/localcache" "github.com/openimsdk/open-im-server/v3/pkg/localcache/lru" "github.com/openimsdk/open-im-server/v3/pkg/rpcclient" "github.com/openimsdk/open-im-server/v3/pkg/util/useronline" + "github.com/openimsdk/tools/db/cacheutil" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/redis/go-redis/v9" - "math/rand" - "strconv" - "time" ) -func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) *OnlineCache { +func NewOnlineCache(user rpcclient.UserRpcClient, group *GroupLocalCache, rdb redis.UniversalClient, fullUserCache bool, fn func(ctx context.Context, userID string, platformIDs []int32)) (*OnlineCache, error) { + l := &sync.Mutex{} x := &OnlineCache{ - user: user, - group: group, - local: lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { + user: user, + group: group, + fullUserCache: fullUserCache, + Lock: l, + Cond: sync.NewCond(l), + } + + ctx := mcontext.SetOperationID(context.TODO(), strconv.FormatInt(time.Now().UnixNano()+int64(rand.Uint32()), 10)) + + switch x.fullUserCache { + case true: + log.ZDebug(ctx, "fullUserCache is true") + x.mapCache = cacheutil.NewCache[string, []int32]() + go func() { + if err := x.initUsersOnlineStatus(ctx); err != nil { + log.ZError(ctx, "initUsersOnlineStatus failed", err) + } + }() + case false: + log.ZDebug(ctx, "fullUserCache is false") + x.lruCache = lru.NewSlotLRU(1024, localcache.LRUStringHash, func() lru.LRU[string, []int32] { return lru.NewLayLRU[string, []int32](2048, cachekey.OnlineExpire/2, time.Second*3, localcache.EmptyTarget{}, func(key string, value []int32) {}) - }), + }) + x.CurrentPhase.Store(DoSubscribeOver) + x.Cond.Broadcast() } + go func() { - ctx := mcontext.SetOperationID(context.Background(), cachekey.OnlineChannel+strconv.FormatUint(rand.Uint64(), 10)) - for message := range rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() { - userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + x.doSubscribe(ctx, rdb, fn) + }() + return x, nil +} + +const ( + Begin uint32 = iota + DoOnlineStatusOver + DoSubscribeOver +) + +type OnlineCache struct { + user rpcclient.UserRpcClient + group *GroupLocalCache + + // fullUserCache if enabled, caches the online status of all users using mapCache; + // otherwise, only a portion of users' online statuses (regardless of whether they are online) will be cached using lruCache. + fullUserCache bool + + lruCache lru.LRU[string, []int32] + mapCache *cacheutil.Cache[string, []int32] + + Lock *sync.Mutex + Cond *sync.Cond + CurrentPhase atomic.Uint32 +} + +func (o *OnlineCache) initUsersOnlineStatus(ctx context.Context) (err error) { + log.ZDebug(ctx, "init users online status begin") + + var ( + totalSet atomic.Int64 + maxTries = 5 + retryInterval = time.Second * 5 + + resp *user.GetAllOnlineUsersResp + ) + + defer func(t time.Time) { + log.ZInfo(ctx, "init users online status end", "cost", time.Since(t), "totalSet", totalSet.Load()) + o.CurrentPhase.Store(DoOnlineStatusOver) + o.Cond.Broadcast() + }(time.Now()) + + retryOperation := func(operation func() error, operationName string) error { + for i := 0; i < maxTries; i++ { + if err = operation(); err != nil { + log.ZWarn(ctx, fmt.Sprintf("initUsersOnlineStatus: %s failed", operationName), err) + time.Sleep(retryInterval) + } else { + return nil + } + } + return err + } + + cursor := uint64(0) + for resp == nil || resp.NextCursor != 0 { + if err = retryOperation(func() error { + resp, err = o.user.GetAllOnlineUsers(ctx, cursor) if err != nil { - log.ZError(ctx, "OnlineCache setUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) - continue + return err + } + + for _, u := range resp.StatusList { + if u.Status == constant.Online { + o.setUserOnline(u.UserID, u.PlatformIDs) + } + totalSet.Add(1) } - storageCache := x.setUserOnline(userID, platformIDs) - log.ZDebug(ctx, "OnlineCache setUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) + cursor = resp.NextCursor + return nil + }, "getAllOnlineUsers"); err != nil { + return err + } + } + + return nil +} + +func (o *OnlineCache) doSubscribe(ctx context.Context, rdb redis.UniversalClient, fn func(ctx context.Context, userID string, platformIDs []int32)) { + o.Lock.Lock() + ch := rdb.Subscribe(ctx, cachekey.OnlineChannel).Channel() + for o.CurrentPhase.Load() < DoOnlineStatusOver { + o.Cond.Wait() + } + o.Lock.Unlock() + log.ZInfo(ctx, "begin doSubscribe") + + doMessage := func(message *redis.Message) { + userID, platformIDs, err := useronline.ParseUserOnlineStatus(message.Payload) + if err != nil { + log.ZError(ctx, "OnlineCache setHasUserOnline redis subscribe parseUserOnlineStatus", err, "payload", message.Payload, "channel", message.Channel) + return + } + log.ZDebug(ctx, fmt.Sprintf("get subscribe %s message", cachekey.OnlineChannel), "useID", userID, "platformIDs", platformIDs) + switch o.fullUserCache { + case true: + if len(platformIDs) == 0 { + // offline + o.mapCache.Delete(userID) + } else { + o.mapCache.Store(userID, platformIDs) + } + case false: + storageCache := o.setHasUserOnline(userID, platformIDs) + log.ZDebug(ctx, "OnlineCache setHasUserOnline", "userID", userID, "platformIDs", platformIDs, "payload", message.Payload, "storageCache", storageCache) if fn != nil { fn(ctx, userID, platformIDs) } } - }() - return x -} + } -type OnlineCache struct { - user rpcclient.UserRpcClient - group *GroupLocalCache - local lru.LRU[string, []int32] + if o.CurrentPhase.Load() == DoOnlineStatusOver { + for done := false; !done; { + select { + case message := <-ch: + doMessage(message) + default: + o.CurrentPhase.Store(DoSubscribeOver) + o.Cond.Broadcast() + done = true + } + } + } + + for message := range ch { + doMessage(message) + } } func (o *OnlineCache) getUserOnlinePlatform(ctx context.Context, userID string) ([]int32, error) { - platformIDs, err := o.local.Get(userID, func() ([]int32, error) { + platformIDs, err := o.lruCache.Get(userID, func() ([]int32, error) { return o.user.GetUserOnlinePlatform(ctx, userID) }) if err != nil { log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userID) return nil, err } - log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) + //log.ZDebug(ctx, "OnlineCache GetUserOnlinePlatform", "userID", userID, "platformIDs", platformIDs) return platformIDs, nil } @@ -69,6 +206,16 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } +// func (o *OnlineCache) GetUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string]int32, error) { +// platformIDs, err := o.getUserOnlinePlatform(ctx, userIDs) +// if err != nil { +// return nil, err +// } +// tmp := make([]int32, len(platformIDs)) +// copy(tmp, platformIDs) +// return platformIDs, nil +// } + func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { @@ -77,10 +224,68 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e return len(platformIDs) > 0, nil } +func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { + platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { + platformIDsMap := make(map[string][]int32) + + usersStatus, err := o.user.GetUsersOnlinePlatform(ctx, missingUsers) + if err != nil { + return nil, err + } + + for _, u := range usersStatus { + platformIDsMap[u.UserID] = u.PlatformIDs + } + + return platformIDsMap, nil + }) + if err != nil { + log.ZError(ctx, "OnlineCache GetUserOnlinePlatform", err, "userID", userIDs) + return nil, err + } + return platformIDsMap, nil +} + +func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, []string, error) { + t := time.Now() + + var ( + onlineUserIDs = make([]string, 0, len(userIDs)) + offlineUserIDs = make([]string, 0, len(userIDs)) + ) + + switch o.fullUserCache { + case true: + for _, userID := range userIDs { + if _, ok := o.mapCache.Load(userID); ok { + onlineUserIDs = append(onlineUserIDs, userID) + } else { + offlineUserIDs = append(offlineUserIDs, userID) + } + } + case false: + userOnlineMap, err := o.getUserOnlinePlatformBatch(ctx, userIDs) + if err != nil { + return nil, nil, err + } + + for key, value := range userOnlineMap { + if len(value) > 0 { + onlineUserIDs = append(onlineUserIDs, key) + } else { + offlineUserIDs = append(offlineUserIDs, key) + } + } + } + + log.ZInfo(ctx, "get users online", "online users length", len(userIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) + return userIDs, offlineUserIDs, nil +} + //func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { // onlineUserIDs := make([]string, 0, len(userIDs)) // for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) +// online, err := o.GetUserOnline(ctx, userID) // if err != nil { // return nil, err // } @@ -111,6 +316,15 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e // return onlineUserIDs, nil //} -func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) bool { - return o.local.SetHas(userID, platformIDs) +func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { + switch o.fullUserCache { + case true: + o.mapCache.Store(userID, platformIDs) + case false: + o.lruCache.Set(userID, platformIDs) + } +} + +func (o *OnlineCache) setHasUserOnline(userID string, platformIDs []int32) bool { + return o.lruCache.SetHas(userID, platformIDs) } diff --git a/pkg/rpcclient/msg.go b/pkg/rpcclient/msg.go index 715014800b..9b26a7abda 100644 --- a/pkg/rpcclient/msg.go +++ b/pkg/rpcclient/msg.go @@ -17,6 +17,11 @@ package rpcclient import ( "context" "encoding/json" + "time" + + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/msg" @@ -28,9 +33,6 @@ import ( "github.com/openimsdk/tools/utils/idutil" "github.com/openimsdk/tools/utils/jsonutil" "github.com/openimsdk/tools/utils/timeutil" - "google.golang.org/grpc" - "google.golang.org/protobuf/proto" - "time" ) func newContentTypeConf(conf *config.Notification) map[int32]config.NotificationConfig { @@ -221,6 +223,19 @@ func (m *MessageRpcClient) PullMessageBySeqList(ctx context.Context, req *sdkws. return resp, nil } +func (m *MessageRpcClient) GetConversationsHasReadAndMaxSeq(ctx context.Context, req *msg.GetConversationsHasReadAndMaxSeqReq) (*msg.GetConversationsHasReadAndMaxSeqResp, error) { + resp, err := m.Client.GetConversationsHasReadAndMaxSeq(ctx, req) + if err != nil { + // Wrap the error to provide more context if the gRPC call fails. + return nil, err + } + return resp, nil +} + +func (m *MessageRpcClient) GetSeqMessage(ctx context.Context, req *msg.GetSeqMessageReq) (*msg.GetSeqMessageResp, error) { + return m.Client.GetSeqMessage(ctx, req) +} + func (m *MessageRpcClient) GetConversationMaxSeq(ctx context.Context, conversationID string) (int64, error) { resp, err := m.Client.GetConversationMaxSeq(ctx, &msg.GetConversationMaxSeqReq{ConversationID: conversationID}) if err != nil { diff --git a/pkg/rpcclient/user.go b/pkg/rpcclient/user.go index eabe77b942..375cc993cb 100644 --- a/pkg/rpcclient/user.go +++ b/pkg/rpcclient/user.go @@ -169,6 +169,15 @@ func (u *UserRpcClient) Access(ctx context.Context, ownerUserID string) error { return authverify.CheckAccessV3(ctx, ownerUserID, u.imAdminUserID) } +// GetAllUserID retrieves all user IDs with pagination options. +func (u *UserRpcClient) GetAllUserID(ctx context.Context, pageNumber, showNumber int32) (*user.GetAllUserIDResp, error) { + resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) + if err != nil { + return nil, err + } + return resp, nil +} + // GetAllUserIDs retrieves all user IDs with pagination options. func (u *UserRpcClient) GetAllUserIDs(ctx context.Context, pageNumber, showNumber int32) ([]string, error) { resp, err := u.Client.GetAllUserID(ctx, &user.GetAllUserIDReq{Pagination: &sdkws.RequestPagination{PageNumber: pageNumber, ShowNumber: showNumber}}) @@ -215,3 +224,7 @@ func (u *UserRpcClient) GetUserOnlinePlatform(ctx context.Context, userID string } return resp[0].PlatformIDs, nil } + +func (u *UserRpcClient) GetAllOnlineUsers(ctx context.Context, cursor uint64) (*user.GetAllOnlineUsersResp, error) { + return u.Client.GetAllOnlineUsers(ctx, &user.GetAllOnlineUsersReq{Cursor: cursor}) +} diff --git a/pkg/util/conversationutil/conversationutil.go b/pkg/util/conversationutil/conversationutil.go index 5683d8df89..f0a44ab1e1 100644 --- a/pkg/util/conversationutil/conversationutil.go +++ b/pkg/util/conversationutil/conversationutil.go @@ -19,6 +19,14 @@ func GenGroupConversationID(groupID string) string { return "sg_" + groupID } +func IsGroupConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "sg_") +} + +func IsNotificationConversationID(conversationID string) bool { + return strings.HasPrefix(conversationID, "n_") +} + func GenConversationUniqueKeyForSingle(sendID, recvID string) string { l := []string{sendID, recvID} sort.Strings(l) diff --git a/scripts/create-topic.sh b/scripts/create-topic.sh index 206075fb83..bbc739287f 100755 --- a/scripts/create-topic.sh +++ b/scripts/create-topic.sh @@ -35,7 +35,7 @@ done echo "Kafka is ready. Creating topics..." -topics=("toRedis" "toMongo" "toPush") +topics=("toRedis" "toMongo" "toPush" "toOfflinePush") partitions=8 replicationFactor=1 diff --git a/scripts/githooks/commit-msg.sh b/scripts/githooks/commit-msg.sh deleted file mode 100644 index d2d96645bc..0000000000 --- a/scripts/githooks/commit-msg.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -# Copyright © 2023 OpenIMSDK. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ============================================================================== -# -# Store this file as .git/hooks/commit-msg in your repository in order to -# enforce checking for proper commit message format before actual commits. -# You may need to make the scripts executable by 'chmod +x .git/hooks/commit-msg'. - -# commit-msg use go-gitlint tool, install go-gitlint via `go get github.com/llorllale/go-gitlint/cmd/go-gitlint` -# go-gitlint --msg-file="$1" - -# An example hook scripts to check the commit log message. -# Called by "git commit" with one argument, the name of the file -# that has the commit message. The hook should exit with non-zero -# status after issuing an appropriate message if it wants to stop the -# commit. The hook is allowed to edit the commit message file. - -YELLOW="\e[93m" -GREEN="\e[32m" -RED="\e[31m" -ENDCOLOR="\e[0m" - -printMessage() { - printf "${YELLOW}OpenIM : $1${ENDCOLOR}\n" -} - -printSuccess() { - printf "${GREEN}OpenIM : $1${ENDCOLOR}\n" -} - -printError() { - printf "${RED}OpenIM : $1${ENDCOLOR}\n" -} - -printMessage "Running the OpenIM commit-msg hook." - -# This example catches duplicate Signed-off-by lines. - -test "" = "$(grep '^Signed-off-by: ' "$1" | -sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || { -echo >&2 Duplicate Signed-off-by lines. -exit 1 -} - -# TODO: go-gitlint dir set -OPENIM_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. -GITLINT_DIR="$OPENIM_ROOT/_output/tools/go-gitlint" - -$GITLINT_DIR \ ---msg-file=$1 \ ---subject-regex="^(build|chore|ci|docs|feat|feature|fix|perf|refactor|revert|style|bot|test)(.*)?:\s?.*" \ ---subject-maxlen=150 \ ---subject-minlen=10 \ ---body-regex=".*" \ ---max-parents=1 - -if [ $? -ne 0 ] -then -if ! command -v $GITLINT_DIR &>/dev/null; then - printError "$GITLINT_DIR not found. Please run 'make tools' OR 'make tools.verify.go-gitlint' make verto install it." -fi -printError "Please fix your commit message to match kubecub coding standards" -printError "https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694#file-githook-md" -exit 1 -fi - -### Add Sign-off-by line to the end of the commit message -# Get local git config -NAME=$(git config user.name) -EMAIL=$(git config user.email) - -# Check if the commit message contains a sign-off line -grep -qs "^Signed-off-by: " "$1" -SIGNED_OFF_BY_EXISTS=$? - -# Add "Signed-off-by" line if it doesn't exist -if [ $SIGNED_OFF_BY_EXISTS -ne 0 ]; then -echo -e "\nSigned-off-by: $NAME <$EMAIL>" >> "$1" -fi \ No newline at end of file diff --git a/scripts/githooks/pre-commit.sh b/scripts/githooks/pre-commit.sh deleted file mode 100644 index d8396b5605..0000000000 --- a/scripts/githooks/pre-commit.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# Copyright © 2023 OpenIMSDK. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ============================================================================== -# This is a pre-commit hook that ensures attempts to commit files that are -# are larger than $limit to your _local_ repo fail, with a helpful error message. - -# You can override the default limit of 2MB by supplying the environment variable: -# GIT_FILE_SIZE_LIMIT=50000000 git commit -m "test: this commit is allowed file sizes up to 50MB" -# -# ============================================================================== -# - -LC_ALL=C - -local_branch="$(git rev-parse --abbrev-ref HEAD)" -valid_branch_regex="^(main|master|develop|release(-[a-zA-Z0-9._-]+)?)$|(feature|feat|openim|hotfix|test|bug|bot|refactor|revert|ci|cicd|style|)\/[a-z0-9._-]+$|^HEAD$" - -YELLOW="\e[93m" -GREEN="\e[32m" -RED="\e[31m" -ENDCOLOR="\e[0m" - -printMessage() { - printf "${YELLOW}openim : $1${ENDCOLOR}\n" -} - -printSuccess() { - printf "${GREEN}openim : $1${ENDCOLOR}\n" -} - -printError() { - printf "${RED}openim : $1${ENDCOLOR}\n" -} - -printMessage "Running local openim pre-commit hook." - -# flutter format . -# https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694#file-githook-md -# TODO! GIT_FILE_SIZE_LIMIT=50000000 git commit -m "test: this commit is allowed file sizes up to 50MB" -# Maximum file size limit in bytes -limit=${GIT_FILE_SIZE_LIMIT:-2000000} # Default 2MB -limitInMB=$(( $limit / 1000000 )) - -function file_too_large(){ - filename=$0 - filesize=$(( $1 / 2**20 )) - - cat < /dev/null 2>&1 -then - against=HEAD -else - against="$empty_tree" -fi - -# Set split so that for loop below can handle spaces in file names by splitting on line breaks -IFS=' -' - -shouldFail=false -for file in $( git diff-index --cached --name-only $against ); do - file_size=$(([ ! -f $file ] && echo 0) || (ls -la $file | awk '{ print $5 }')) - if [ "$file_size" -gt "$limit" ]; then - printError "File $file is $(( $file_size / 10**6 )) MB, which is larger than our configured limit of $limitInMB MB" - shouldFail=true - fi -done - -if $shouldFail -then - printMessage "If you really need to commit this file, you can override the size limit by setting the GIT_FILE_SIZE_LIMIT environment variable, e.g. GIT_FILE_SIZE_LIMIT=42000000 for 42MB. Or, commit with the --no-verify switch to skip the check entirely." - printError "Commit aborted" - exit 1; -fi - -if [[ ! $local_branch =~ $valid_branch_regex ]] -then - printError "There is something wrong with your branch name. Branch names in this project must adhere to this contract: $valid_branch_regex. -Your commit will be rejected. You should rename your branch to a valid name(feat/name OR fix/name) and try again." - printError "For more on this, read on: https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694" - exit 1 -fi \ No newline at end of file diff --git a/scripts/githooks/pre-push.sh b/scripts/githooks/pre-push.sh deleted file mode 100644 index 9bd9389158..0000000000 --- a/scripts/githooks/pre-push.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash -# Copyright © 2023 OpenIMSDK. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ============================================================================== -# - -YELLOW="\e[93m" -GREEN="\e[32m" -RED="\e[31m" -ENDCOLOR="\e[0m" - -local_branch="$(git rev-parse --abbrev-ref HEAD)" -valid_branch_regex="^(main|master|develop|release(-[a-zA-Z0-9._-]+)?)$|(feature|feat|openim|hotfix|test|bug|ci|cicd|style|)\/[a-z0-9._-]+$|^HEAD$" - -printMessage() { - printf "${YELLOW}OpenIM : $1${ENDCOLOR}\n" -} - -printSuccess() { - printf "${GREEN}OpenIM : $1${ENDCOLOR}\n" -} - -printError() { - printf "${RED}OpenIM : $1${ENDCOLOR}\n" -} - -printMessage "Running local OpenIM pre-push hook." - -if [[ $(git status --porcelain) ]]; then - printError "This scripts needs to run against committed code only. Please commit or stash you changes." - exit 1 -fi - -COLOR_SUFFIX="\033[0m" - -BLACK_PREFIX="\033[30m" -RED_PREFIX="\033[31m" -GREEN_PREFIX="\033[32m" -BACKGROUND_GREEN="\033[33m" -BLUE_PREFIX="\033[34m" -PURPLE_PREFIX="\033[35m" -SKY_BLUE_PREFIX="\033[36m" -WHITE_PREFIX="\033[37m" -BOLD_PREFIX="\033[1m" -UNDERLINE_PREFIX="\033[4m" -ITALIC_PREFIX="\033[3m" - -# Function to print colored text -print_color() { - local text=$1 - local color=$2 - echo -e "${color}${text}${COLOR_SUFFIX}" -} - -# Function to print section separator -print_separator() { - print_color "==========================================================" ${PURPLE_PREFIX} -} - -# Get current time -time=$(date +"%Y-%m-%d %H:%M:%S") - -# Print section separator -print_separator - -# Print time of submission -print_color "PTIME: ${time}" "${BOLD_PREFIX}${CYAN_PREFIX}" -echo "" -author=$(git config user.name) -repository=$(basename -s .git $(git config --get remote.origin.url)) - -# Print additional information if needed -print_color "Repository: ${repository}" "${BLUE_PREFIX}" -echo "" - -print_color "Author: ${author}" "${PURPLE_PREFIX}" - -# Print section separator -print_separator - -file_list=$(git diff --name-status HEAD @{u}) -added_files=$(grep -c '^A' <<< "$file_list") -modified_files=$(grep -c '^M' <<< "$file_list") -deleted_files=$(grep -c '^D' <<< "$file_list") - -print_color "Added Files: ${added_files}" "${BACKGROUND_GREEN}" -print_color "Modified Files: ${modified_files}" "${BACKGROUND_GREEN}" -print_color "Deleted Files: ${deleted_files}" "${BACKGROUND_GREEN}" - -if [[ ! $local_branch =~ $valid_branch_regex ]] -then - printError "There is something wrong with your branch name. Branch names in this project must adhere to this contract: $valid_branch_regex. -Your commit will be rejected. You should rename your branch to a valid name(feat/name OR fix/name) and try again." - printError "For more on this, read on: https://gist.github.com/cubxxw/126b72104ac0b0ca484c9db09c3e5694" - exit 1 -fi - -# -#printMessage "Running the Flutter analyzer" -#flutter analyze -# -#if [ $? -ne 0 ]; then -# printError "Flutter analyzer error" -# exit 1 -#fi -# -#printMessage "Finished running the Flutter analyzer" diff --git a/start-config.yml b/start-config.yml index a6d3e47afa..1231b5d0d4 100644 --- a/start-config.yml +++ b/start-config.yml @@ -3,8 +3,8 @@ serviceBinaries: openim-crontask: 1 openim-rpc-user: 1 openim-msggateway: 1 - openim-push: 4 - openim-msgtransfer: 4 + openim-push: 8 + openim-msgtransfer: 8 openim-rpc-conversation: 1 openim-rpc-auth: 1 openim-rpc-group: 1 diff --git a/test/e2e/api/token/token.go b/test/e2e/api/token/token.go index 908f9b8a0f..c862dc6db4 100644 --- a/test/e2e/api/token/token.go +++ b/test/e2e/api/token/token.go @@ -53,8 +53,7 @@ type User struct { // UserRegisterRequest represents a request to register a user. type UserRegisterRequest struct { - Secret string `json:"secret"` - Users []User `json:"users"` + Users []User `json:"users"` } /* func main() { @@ -109,8 +108,7 @@ func RegisterUser(token, userID, nickname, faceURL string) error { FaceURL: faceURL, } reqBody := UserRegisterRequest{ - Secret: SecretKey, - Users: []User{user}, + Users: []User{user}, } reqBytes, err := json.Marshal(reqBody) if err != nil { diff --git a/tools/check-component/main.go b/tools/check-component/main.go index 5fa84ac36f..4f4c08c16a 100644 --- a/tools/check-component/main.go +++ b/tools/check-component/main.go @@ -18,6 +18,12 @@ import ( "context" "flag" "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/tools/db/mongoutil" @@ -27,11 +33,6 @@ import ( "github.com/openimsdk/tools/mq/kafka" "github.com/openimsdk/tools/s3/minio" "github.com/openimsdk/tools/system/program" - "io/ioutil" - "log" - "os" - "path/filepath" - "time" ) const maxRetry = 180 @@ -65,7 +66,7 @@ func CheckMinIO(ctx context.Context, config *config.Minio) error { } func CheckKafka(ctx context.Context, conf *config.Kafka) error { - return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic}) + return kafka.Check(ctx, conf.Build(), []string{conf.ToMongoTopic, conf.ToRedisTopic, conf.ToPushTopic, conf.ToOfflinePushTopic}) } func initConfig(configDir string) (*config.Mongo, *config.Redis, *config.Kafka, *config.Minio, *config.Discovery, error) { From c8dd598fbca176b254141b0773ed0aff82ec2891 Mon Sep 17 00:00:00 2001 From: Monet Lee Date: Thu, 26 Sep 2024 17:19:30 +0800 Subject: [PATCH 06/12] update go mod --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 62cb5b7b17..1095134dd3 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,6 @@ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= -github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= From 9fb5ee5b554bdb5de557f245d86324ead1297dd7 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:54:27 +0800 Subject: [PATCH 07/12] merge: update code from main to v3.8-js-sdk-only. (#2720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: fix update groupName invalid. (#2673) * refactor: change platform to platformID (#2670) * feat: don`t return nil data (#2675) Co-authored-by: Monet Lee * refactor: update fields type in userStatus and check registered. (#2676) * fix: usertoken auth. (#2677) * refactor: update fields type in userStatus and check registered. * fix: usertoken auth. * update contents. * update content. * update * fix * update pb file. * feat: add friend agree after callback (#2680) * fix: sn not sort (#2682) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort --------- Co-authored-by: withchao * refactor: add GetAdminToken interface. (#2684) * refactor: add GetAdminToken interface. * update config. * fix: admin token (#2686) * fix: update workflows logic. (#2688) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * fix: admin token (#2687) * update the front image (#2692) * update the front image * update version * feat: improve publish docker image workflows (#2697) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. (#2700) * refactor: add GetAdminToken interface. * update config. * update workflows logic. * feat: improve publish docker image workflows * update condition logic. * fix: update load file logic. * feat: Msg filter (#2703) * feat: msg filter * feat: msg filter * feat: msg filter * feat: provide the interface required by js sdk (#2712) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support --------- Co-authored-by: withchao * Line webhook (#2716) * feat: online and offline webhook * feat: online and offline webhook * feat: remove zk * fix: the message I sent is not set to read seq in mongodb (#2718) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb --------- Co-authored-by: withchao * fix: cannot modify group member avatars (#2719) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars --------- Co-authored-by: withchao --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao Co-authored-by: skiffer-git <72860476+skiffer-git@users.noreply.github.com> --- .env | 8 +- .github/workflows/publish-docker-image.yml | 89 ++--- Dockerfile | 2 +- cmd/openim-api/main.go | 3 +- config/discovery.yml | 5 - config/webhooks.yml | 27 +- docker-compose.yml | 14 +- go.mod | 4 +- go.sum | 8 +- internal/api/auth.go | 4 +- internal/api/jssdk/jssdk.go | 122 +++++-- internal/api/jssdk/stu.go | 22 -- internal/api/jssdk/tools.go | 57 ++- internal/api/jssdk_test.go | 37 -- internal/api/router.go | 7 +- internal/api/user.go | 16 +- internal/msggateway/hub_server.go | 10 +- internal/msggateway/online.go | 13 + internal/push/a_test.go | 29 -- internal/rpc/auth/auth.go | 20 +- internal/rpc/group/callback.go | 2 +- internal/rpc/group/db_map.go | 8 +- internal/rpc/group/group.go | 4 +- internal/rpc/msg/callback.go | 16 +- internal/rpc/msg/filter.go | 67 ++++ internal/rpc/relation/callback.go | 12 + internal/rpc/relation/friend.go | 10 +- internal/rpc/user/online.go | 3 +- pkg/callbackstruct/constant.go | 1 + pkg/callbackstruct/friend.go | 12 + pkg/common/cmd/msg_gateway_test.go | 9 +- pkg/common/config/config.go | 11 +- pkg/common/convert/friend.go | 25 +- pkg/common/storage/controller/auth.go | 12 + pkg/common/storage/controller/msg_transfer.go | 2 +- pkg/msgprocessor/conversation.go | 72 +--- pkg/msgprocessor/conversation_test.go | 334 ------------------ tools/url2im/pkg/api.go | 16 +- tools/url2im/pkg/manage.go | 5 +- version/version | 2 +- 40 files changed, 466 insertions(+), 654 deletions(-) delete mode 100644 internal/api/jssdk/stu.go delete mode 100644 internal/api/jssdk_test.go delete mode 100644 internal/push/a_test.go create mode 100644 internal/rpc/msg/filter.go delete mode 100644 pkg/msgprocessor/conversation_test.go diff --git a/.env b/.env index 8863f2ec1e..0c02bf9f6b 100644 --- a/.env +++ b/.env @@ -8,12 +8,12 @@ PROMETHEUS_IMAGE=prom/prometheus:v2.45.6 ALERTMANAGER_IMAGE=prom/alertmanager:v0.27.0 GRAFANA_IMAGE=grafana/grafana:11.0.1 -OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.0 -OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.0 +OPENIM_WEB_FRONT_IMAGE=openim/openim-web-front:release-v3.8.1 +OPENIM_ADMIN_FRONT_IMAGE=openim/openim-admin-front:release-v1.8.2 #FRONT_IMAGE: use aliyun images -#OPENIM_WEB_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web-front:release-v3.5.1 -#OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.7 +#OPENIM_WEB_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-web-front:release-v3.8.1 +#OPENIM_ADMIN_FRONT_IMAGE=registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-admin-front:release-v1.8.2 DATA_DIR=./ diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 7d7d23f173..40b79e61ad 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -4,6 +4,8 @@ on: push: branches: - release-* + # tags: + # - 'v*' release: types: [published] @@ -15,11 +17,8 @@ on: required: true default: "v3.8.0" -# env: -# GO_VERSION: "1.21" - jobs: - publish-docker-images: + build-and-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -28,16 +27,22 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v3 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and push Docker image + - name: Build Docker image + id: build uses: docker/build-push-action@v5 with: context: ./main-repo load: true tags: "openim/openim-server:local" + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Save Docker image to file + run: docker save -o image.tar openim/openim-server:local - name: Checkout compose repository uses: actions/checkout@v4 @@ -63,38 +68,39 @@ jobs: docker compose up -d sleep 60 - - name: Check openim-server health - run: | - timeout=300 - interval=30 - elapsed=0 - while [[ $elapsed -le $timeout ]]; do - if ! docker exec openim-server mage check; then - echo "openim-server is not ready, waiting..." - sleep $interval - elapsed=$(($elapsed + $interval)) - else - echo "Health check successful" - exit 0 - fi - done - echo "Health check failed after 5 minutes" - exit 1 - - - name: Check openim-chat health - if: success() - run: | - if ! docker exec openim-chat mage check; then - echo "openim-chat check failed" - exit 1 - else - echo "Health check successful" - exit 0 - fi - - - - name: Extract metadata for Docker # (tags, labels) - if: success() + # - name: Check openim-server health + # run: | + # timeout=300 + # interval=30 + # elapsed=0 + # while [[ $elapsed -le $timeout ]]; do + # if ! docker exec openim-server mage check; then + # echo "openim-server is not ready, waiting..." + # sleep $interval + # elapsed=$(($elapsed + $interval)) + # else + # echo "Health check successful" + # exit 0 + # fi + # done + # echo "Health check failed after 5 minutes" + # exit 1 + + # - name: Check openim-chat health + # if: success() + # run: | + # if ! docker exec openim-chat mage check; then + # echo "openim-chat check failed" + # exit 1 + # else + # echo "Health check successful" + # exit 0 + # fi + + - name: Load Docker image from file + run: docker load -i image.tar + + - name: Extract metadata for Docker (tags, labels) id: meta uses: docker/metadata-action@v5.5.1 with: @@ -102,18 +108,17 @@ jobs: openim/openim-server ghcr.io/openimsdk/openim-server registry.cn-hangzhou.aliyuncs.com/openimsdk/openim-server - - # generate Docker tags based on the following events/attributes tags: | type=ref,event=tag type=schedule type=ref,event=branch - type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern=v{{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} + type=semver,pattern=release-{{raw}} type=sha + type=raw,value=${{ github.event.inputs.tag }} - name: Log in to Docker Hub uses: docker/login-action@v2 @@ -135,7 +140,7 @@ jobs: username: ${{ secrets.ALIREGISTRY_USERNAME }} password: ${{ secrets.ALIREGISTRY_TOKEN }} - - name: Build and push Docker images + - name: Push Docker images uses: docker/build-push-action@v5 with: context: ./main-repo diff --git a/Dockerfile b/Dockerfile index e082dd64c8..f8cfbda9ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use Go 1.21 Alpine as the base image for building the application -FROM golang:1.21-alpine as builder +FROM golang:1.21-alpine AS builder # Define the base directory for the application as an environment variable ENV SERVER_DIR=/openim-server diff --git a/cmd/openim-api/main.go b/cmd/openim-api/main.go index e29ed2a592..3690cfc991 100644 --- a/cmd/openim-api/main.go +++ b/cmd/openim-api/main.go @@ -15,10 +15,9 @@ package main import ( - _ "net/http/pprof" - "github.com/openimsdk/open-im-server/v3/pkg/common/cmd" "github.com/openimsdk/tools/system/program" + _ "net/http/pprof" ) func main() { diff --git a/config/discovery.yml b/config/discovery.yml index 78a36f3d1f..6e68cbff46 100644 --- a/config/discovery.yml +++ b/config/discovery.yml @@ -5,9 +5,4 @@ etcd: username: '' password: '' -zookeeper: - schema: openim - address: [ localhost:12181 ] - username: '' - password: '' diff --git a/config/webhooks.yml b/config/webhooks.yml index eee79c650e..854d2dc2cc 100644 --- a/config/webhooks.yml +++ b/config/webhooks.yml @@ -1,8 +1,18 @@ -url: webhook://127.0.0.1:10008/callbackExample +url: http://127.0.0.1:10006/callbackExample beforeSendSingleMsg: enable: false timeout: 5 failedContinue: true + # Only the contentType in allowedTypes will send the callback. + # Supports two formats: a single type or a range. The range is defined by the lower and upper bounds connected with a hyphen ("-"). + # e.g. allowedTypes: [1, 100, 200-500, 600-700] means that only contentType within the range + # {1, 100} ∪ [200, 500] ∪ [600, 700] will be allowed through the filter. + # If not set, all contentType messages will through this filter. + allowedTypes: [] + # Only the contentType not in deniedTypes will send the callback. + # Supports two formats, same as allowedTypes. + # If not set, all contentType messages will through this filter. + deniedTypes: [] beforeUpdateUserInfoEx: enable: false timeout: 5 @@ -16,17 +26,29 @@ afterSendSingleMsg: # Only the senID/recvID specified in attentionIds will send the callback # if not set, all user messages will be callback attentionIds: [] + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] beforeSendGroupMsg: enable: false timeout: 5 failedContinue: true + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] beforeMsgModify: enable: false timeout: 5 failedContinue: true + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] afterSendGroupMsg: enable: false timeout: 5 + # See beforeSendSingleMsg comment. + allowedTypes: [] + deniedTypes: [] afterUserOnline: enable: false timeout: 5 @@ -151,6 +173,9 @@ beforeAddFriendAgree: enable: false timeout: 5 failedContinue: true +afterAddFriendAgree: + enable: false + timeout: 5 afterDeleteFriend: enable: false timeout: 5 diff --git a/docker-compose.yml b/docker-compose.yml index edac65b13e..6d88bac10e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,19 +43,6 @@ services: networks: - openim - zookeeper: - image: "${ZOOKEEPER_IMAGE}" - container_name: zookeeper - ports: - - "12181:2181" - environment: - #JVMFLAGS: "-Xms32m -Xmx128m" - TZ: "Asia/Shanghai" - ALLOW_ANONYMOUS_LOGIN: "yes" - restart: always - networks: - - openim - etcd: image: "${ETCD_IMAGE}" container_name: etcd @@ -142,6 +129,7 @@ services: # image: ${PROMETHEUS_IMAGE} # container_name: prometheus # restart: always +# user: root # volumes: # - ./config/prometheus.yml:/etc/prometheus/prometheus.yml # - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml diff --git a/go.mod b/go.mod index 5cc1f9ad3c..b6baca2a13 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.30 - github.com/openimsdk/tools v0.0.50-alpha.12 + github.com/openimsdk/protocol v0.0.72-alpha.41 + github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 1095134dd3..6f54752741 100644 --- a/go.sum +++ b/go.sum @@ -319,10 +319,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.30 h1:LBIqDzD55cSQy3wX8fgSa3blz8+Cv54ae96/qUMINwM= -github.com/openimsdk/protocol v0.0.72-alpha.30/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= -github.com/openimsdk/tools v0.0.50-alpha.12 h1:rV3BxgqN+F79vZvdoQ+97Eob8ScsRVEM8D+Wrcl23uo= -github.com/openimsdk/tools v0.0.50-alpha.12/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= +github.com/openimsdk/protocol v0.0.72-alpha.41 h1:SMMoTc1iu+wtRqUqmIgqPJFejLgPeauOwoJ4VVG4iMQ= +github.com/openimsdk/protocol v0.0.72-alpha.41/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= +github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/api/auth.go b/internal/api/auth.go index f0790ce984..f41b530bf1 100644 --- a/internal/api/auth.go +++ b/internal/api/auth.go @@ -27,8 +27,8 @@ func NewAuthApi(client rpcclient.Auth) AuthApi { return AuthApi(client) } -func (o *AuthApi) UserToken(c *gin.Context) { - a2r.Call(auth.AuthClient.UserToken, o.Client, c) +func (o *AuthApi) GetAdminToken(c *gin.Context) { + a2r.Call(auth.AuthClient.GetAdminToken, o.Client, c) } func (o *AuthApi) GetUserToken(c *gin.Context) { diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 7f136c74ce..2fcbf5ec66 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -1,11 +1,15 @@ package jssdk import ( + "context" "github.com/gin-gonic/gin" "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/group" + "github.com/openimsdk/protocol/jssdk" "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" - "github.com/openimsdk/tools/a2r" + "github.com/openimsdk/protocol/user" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" "sort" @@ -16,16 +20,22 @@ const ( defaultGetActiveConversation = 100 ) -func NewJSSdkApi(msg msg.MsgClient, conv conversation.ConversationClient) *JSSdk { +func NewJSSdkApi(user user.UserClient, friend relation.FriendClient, group group.GroupClient, msg msg.MsgClient, conv conversation.ConversationClient) *JSSdk { return &JSSdk{ - msg: msg, - conv: conv, + user: user, + friend: friend, + group: group, + msg: msg, + conv: conv, } } type JSSdk struct { - msg msg.MsgClient - conv conversation.ConversationClient + user user.UserClient + friend relation.FriendClient + group group.GroupClient + msg msg.MsgClient + conv conversation.ConversationClient } func (x *JSSdk) GetActiveConversations(c *gin.Context) { @@ -36,25 +46,71 @@ func (x *JSSdk) GetConversations(c *gin.Context) { call(c, x.getConversations) } -func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, error) { - req, err := a2r.ParseRequest[ActiveConversationsReq](ctx) - if err != nil { - return nil, err +func (x *JSSdk) fillConversations(ctx context.Context, conversations []*jssdk.ConversationMsg) error { + if len(conversations) == 0 { + return nil + } + var ( + userIDs []string + groupIDs []string + ) + for _, c := range conversations { + if c.Conversation.GroupID == "" { + userIDs = append(userIDs, c.Conversation.UserID) + } else { + groupIDs = append(groupIDs, c.Conversation.GroupID) + } + } + var ( + userMap map[string]*sdkws.UserInfo + friendMap map[string]*relation.FriendInfoOnly + groupMap map[string]*sdkws.GroupInfo + ) + if len(userIDs) > 0 { + users, err := field(ctx, x.user.GetDesignateUsers, &user.GetDesignateUsersReq{UserIDs: userIDs}, (*user.GetDesignateUsersResp).GetUsersInfo) + if err != nil { + return err + } + friends, err := field(ctx, x.friend.GetFriendInfo, &relation.GetFriendInfoReq{OwnerUserID: conversations[0].Conversation.OwnerUserID, FriendUserIDs: userIDs}, (*relation.GetFriendInfoResp).GetFriendInfos) + if err != nil { + return err + } + userMap = datautil.SliceToMap(users, (*sdkws.UserInfo).GetUserID) + friendMap = datautil.SliceToMap(friends, (*relation.FriendInfoOnly).GetFriendUserID) + } + if len(groupIDs) > 0 { + resp, err := x.group.GetGroupsInfo(ctx, &group.GetGroupsInfoReq{GroupIDs: groupIDs}) + if err != nil { + return err + } + groupMap = datautil.SliceToMap(resp.GroupInfos, (*sdkws.GroupInfo).GetGroupID) } + for _, c := range conversations { + if c.Conversation.GroupID == "" { + c.User = userMap[c.Conversation.UserID] + c.Friend = friendMap[c.Conversation.UserID] + } else { + c.Group = groupMap[c.Conversation.GroupID] + } + } + return nil +} + +func (x *JSSdk) getActiveConversations(ctx context.Context, req *jssdk.GetActiveConversationsReq) (*jssdk.GetActiveConversationsResp, error) { if req.Count <= 0 || req.Count > maxGetActiveConversation { req.Count = defaultGetActiveConversation } - opUserID := mcontext.GetOpUserID(ctx) + req.OwnerUserID = mcontext.GetOpUserID(ctx) conversationIDs, err := field(ctx, x.conv.GetConversationIDs, - &conversation.GetConversationIDsReq{UserID: opUserID}, (*conversation.GetConversationIDsResp).GetConversationIDs) + &conversation.GetConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetConversationIDsResp).GetConversationIDs) if err != nil { return nil, err } if len(conversationIDs) == 0 { - return &ConversationsResp{}, nil + return &jssdk.GetActiveConversationsResp{}, nil } readSeq, err := field(ctx, x.msg.GetHasReadSeqs, - &msg.GetHasReadSeqsReq{UserID: opUserID, ConversationIDs: conversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) + &msg.GetHasReadSeqsReq{UserID: req.OwnerUserID, ConversationIDs: conversationIDs}, (*msg.SeqsInfoResp).GetMaxSeqs) if err != nil { return nil, err } @@ -64,24 +120,24 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er return nil, err } if len(activeConversation) == 0 { - return &ConversationsResp{}, nil + return &jssdk.GetActiveConversationsResp{}, nil } sortConversations := sortActiveConversations{ Conversation: activeConversation, } if len(activeConversation) > 1 { pinnedConversationIDs, err := field(ctx, x.conv.GetPinnedConversationIDs, - &conversation.GetPinnedConversationIDsReq{UserID: opUserID}, (*conversation.GetPinnedConversationIDsResp).GetConversationIDs) + &conversation.GetPinnedConversationIDsReq{UserID: req.OwnerUserID}, (*conversation.GetPinnedConversationIDsResp).GetConversationIDs) if err != nil { return nil, err } sortConversations.PinnedConversationIDs = datautil.SliceSet(pinnedConversationIDs) } sort.Sort(&sortConversations) - sortList := sortConversations.Top(req.Count) + sortList := sortConversations.Top(int(req.Count)) conversations, err := field(ctx, x.conv.GetConversations, &conversation.GetConversationsReq{ - OwnerUserID: opUserID, + OwnerUserID: req.OwnerUserID, ConversationIDs: datautil.Slice(sortList, func(c *msg.ActiveConversation) string { return c.ConversationID })}, (*conversation.GetConversationsResp).GetConversations) @@ -90,7 +146,7 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er } msgs, err := field(ctx, x.msg.GetSeqMessage, &msg.GetSeqMessageReq{ - UserID: opUserID, + UserID: req.OwnerUserID, Conversations: datautil.Slice(sortList, func(c *msg.ActiveConversation) *msg.ConversationSeqs { return &msg.ConversationSeqs{ ConversationID: c.ConversationID, @@ -104,7 +160,7 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er conversationMap := datautil.SliceToMap(conversations, func(c *conversation.Conversation) string { return c.ConversationID }) - resp := make([]ConversationMsg, 0, len(sortList)) + resp := make([]*jssdk.ConversationMsg, 0, len(sortList)) for _, c := range sortList { conv, ok := conversationMap[c.ConversationID] if !ok { @@ -114,13 +170,16 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er if msgList, ok := msgs[c.ConversationID]; ok && len(msgList.Msgs) > 0 { lastMsg = msgList.Msgs[0] } - resp = append(resp, ConversationMsg{ + resp = append(resp, &jssdk.ConversationMsg{ Conversation: conv, LastMsg: lastMsg, MaxSeq: c.MaxSeq, ReadSeq: readSeq[c.ConversationID], }) } + if err := x.fillConversations(ctx, resp); err != nil { + return nil, err + } var unreadCount int64 for _, c := range activeConversation { count := c.MaxSeq - readSeq[c.ConversationID] @@ -128,24 +187,20 @@ func (x *JSSdk) getActiveConversations(ctx *gin.Context) (*ConversationsResp, er unreadCount += count } } - return &ConversationsResp{ + return &jssdk.GetActiveConversationsResp{ Conversations: resp, UnreadCount: unreadCount, }, nil } -func (x *JSSdk) getConversations(ctx *gin.Context) (*ConversationsResp, error) { - req, err := a2r.ParseRequest[conversation.GetConversationsReq](ctx) - if err != nil { - return nil, err - } +func (x *JSSdk) getConversations(ctx context.Context, req *jssdk.GetConversationsReq) (*jssdk.GetConversationsResp, error) { req.OwnerUserID = mcontext.GetOpUserID(ctx) - conversations, err := field(ctx, x.conv.GetConversations, req, (*conversation.GetConversationsResp).GetConversations) + conversations, err := field(ctx, x.conv.GetConversations, &conversation.GetConversationsReq{OwnerUserID: req.OwnerUserID, ConversationIDs: req.ConversationIDs}, (*conversation.GetConversationsResp).GetConversations) if err != nil { return nil, err } if len(conversations) == 0 { - return &ConversationsResp{}, nil + return &jssdk.GetConversationsResp{}, nil } req.ConversationIDs = datautil.Slice(conversations, func(c *conversation.Conversation) string { return c.ConversationID @@ -177,19 +232,22 @@ func (x *JSSdk) getConversations(ctx *gin.Context) (*ConversationsResp, error) { return nil, err } } - resp := make([]ConversationMsg, 0, len(conversations)) + resp := make([]*jssdk.ConversationMsg, 0, len(conversations)) for _, c := range conversations { var lastMsg *sdkws.MsgData if msgList, ok := msgs[c.ConversationID]; ok && len(msgList.Msgs) > 0 { lastMsg = msgList.Msgs[0] } - resp = append(resp, ConversationMsg{ + resp = append(resp, &jssdk.ConversationMsg{ Conversation: c, LastMsg: lastMsg, MaxSeq: maxSeqs[c.ConversationID], ReadSeq: readSeqs[c.ConversationID], }) } + if err := x.fillConversations(ctx, resp); err != nil { + return nil, err + } var unreadCount int64 for conversationID, maxSeq := range maxSeqs { count := maxSeq - readSeqs[conversationID] @@ -197,7 +255,7 @@ func (x *JSSdk) getConversations(ctx *gin.Context) (*ConversationsResp, error) { unreadCount += count } } - return &ConversationsResp{ + return &jssdk.GetConversationsResp{ Conversations: resp, UnreadCount: unreadCount, }, nil diff --git a/internal/api/jssdk/stu.go b/internal/api/jssdk/stu.go deleted file mode 100644 index 2f63975b3b..0000000000 --- a/internal/api/jssdk/stu.go +++ /dev/null @@ -1,22 +0,0 @@ -package jssdk - -import ( - "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/sdkws" -) - -type ActiveConversationsReq struct { - Count int `json:"count"` -} - -type ConversationMsg struct { - Conversation *conversation.Conversation `json:"conversation"` - LastMsg *sdkws.MsgData `json:"lastMsg"` - MaxSeq int64 `json:"maxSeq"` - ReadSeq int64 `json:"readSeq"` -} - -type ConversationsResp struct { - UnreadCount int64 `json:"unreadCount"` - Conversations []ConversationMsg `json:"conversations"` -} diff --git a/internal/api/jssdk/tools.go b/internal/api/jssdk/tools.go index c57457d9f4..c19d8970b6 100644 --- a/internal/api/jssdk/tools.go +++ b/internal/api/jssdk/tools.go @@ -3,8 +3,14 @@ package jssdk import ( "context" "github.com/gin-gonic/gin" + "github.com/openimsdk/tools/a2r" "github.com/openimsdk/tools/apiresp" + "github.com/openimsdk/tools/checker" + "github.com/openimsdk/tools/errs" "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + "io" + "strings" ) func field[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) { @@ -16,11 +22,56 @@ func field[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A return get(resp), nil } -func call[R any](c *gin.Context, fn func(ctx *gin.Context) (R, error)) { - resp, err := fn(c) +func call[A, B any](c *gin.Context, fn func(ctx context.Context, req *A) (*B, error)) { + var isJSON bool + switch contentType := c.GetHeader("Content-Type"); { + case contentType == "": + isJSON = true + case strings.Contains(contentType, "application/json"): + isJSON = true + case strings.Contains(contentType, "application/protobuf"): + case strings.Contains(contentType, "application/x-protobuf"): + default: + apiresp.GinError(c, errs.ErrArgs.WrapMsg("unsupported content type")) + return + } + var req *A + if isJSON { + var err error + req, err = a2r.ParseRequest[A](c) + if err != nil { + apiresp.GinError(c, err) + return + } + } else { + body, err := io.ReadAll(c.Request.Body) + if err != nil { + apiresp.GinError(c, err) + return + } + req = new(A) + if err := proto.Unmarshal(body, any(req).(proto.Message)); err != nil { + apiresp.GinError(c, err) + return + } + if err := checker.Validate(&req); err != nil { + apiresp.GinError(c, err) + return + } + } + resp, err := fn(c, req) + if err != nil { + apiresp.GinError(c, err) + return + } + if isJSON { + apiresp.GinSuccess(c, resp) + return + } + body, err := proto.Marshal(any(resp).(proto.Message)) if err != nil { apiresp.GinError(c, err) return } - apiresp.GinSuccess(c, resp) + apiresp.GinSuccess(c, body) } diff --git a/internal/api/jssdk_test.go b/internal/api/jssdk_test.go deleted file mode 100644 index 472ca56b57..0000000000 --- a/internal/api/jssdk_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package api - -import ( - "github.com/openimsdk/protocol/msg" - "sort" - "testing" -) - -func TestName(t *testing.T) { - val := sortActiveConversations{ - Conversation: []*msg.ActiveConversation{ - { - ConversationID: "100", - LastTime: 100, - }, - { - ConversationID: "200", - LastTime: 200, - }, - { - ConversationID: "300", - LastTime: 300, - }, - { - ConversationID: "400", - LastTime: 400, - }, - }, - //PinnedConversationIDs: map[string]struct{}{ - // "100": {}, - // "300": {}, - //}, - } - sort.Sort(&val) - t.Log(val) - -} diff --git a/internal/api/router.go b/internal/api/router.go index 8d2a688f40..560516d303 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "github.com/openimsdk/open-im-server/v3/internal/api/jssdk" "github.com/gin-contrib/gzip" @@ -76,7 +77,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En r.Use(prommetricsGin(), gin.Recovery(), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(authRpc)) u := NewUserApi(*userRpc) m := NewMessageApi(messageRpc, userRpc, config.Share.IMAdminUserID) - j := jssdk.NewJSSdkApi(messageRpc.Client, conversationRpc.Client) + j := jssdk.NewJSSdkApi(userRpc.Client, friendRpc.Client, groupRpc.Client, messageRpc.Client, conversationRpc.Client) userRouterGroup := r.Group("/user") { userRouterGroup.POST("/user_register", u.UserRegister) @@ -168,7 +169,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRouterGroup := r.Group("/auth") { a := NewAuthApi(*authRpc) - authRouterGroup.POST("/user_token", a.UserToken) + authRouterGroup.POST("/get_admin_token", a.GetAdminToken) authRouterGroup.POST("/get_user_token", a.GetUserToken) authRouterGroup.POST("/parse_token", a.ParseToken) authRouterGroup.POST("/force_logout", a.ForceLogout) @@ -287,6 +288,6 @@ func GinParseToken(authRPC *rpcclient.Auth) gin.HandlerFunc { // Whitelist api not parse token var Whitelist = []string{ - "/auth/user_token", + "/auth/get_admin_token", "/auth/parse_token", } diff --git a/internal/api/user.go b/internal/api/user.go index dba7cd3129..b499f71dc2 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -107,14 +107,14 @@ func (u *UserApi) GetUsersOnlineStatus(c *gin.Context) { if v2.UserID == v1 { flag = true res.UserID = v1 - res.Status = constant.OnlineStatus + res.Status = constant.Online res.DetailPlatformStatus = append(res.DetailPlatformStatus, v2.DetailPlatformStatus...) break } } if !flag { res.UserID = v1 - res.Status = constant.OfflineStatus + res.Status = constant.Offline } respResult = append(respResult, res) } @@ -153,26 +153,26 @@ func (u *UserApi) GetUsersOnlineTokenDetail(c *gin.Context) { } for _, v1 := range req.UserIDs { - m := make(map[string][]string, 10) + m := make(map[int32][]string, 10) flag = false temp := new(msggateway.SingleDetail) for _, v2 := range wsResult { if v2.UserID == v1 { flag = true temp.UserID = v1 - temp.Status = constant.OnlineStatus + temp.Status = constant.Online for _, status := range v2.DetailPlatformStatus { - if v, ok := m[status.Platform]; ok { - m[status.Platform] = append(v, status.Token) + if v, ok := m[status.PlatformID]; ok { + m[status.PlatformID] = append(v, status.Token) } else { - m[status.Platform] = []string{status.Token} + m[status.PlatformID] = []string{status.Token} } } } } for p, tokens := range m { t := new(msggateway.SinglePlatformToken) - t.Platform = p + t.PlatformID = p t.Token = tokens t.Total = int32(len(tokens)) temp.SinglePlatformToken = append(temp.SinglePlatformToken, t) diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 01482aa117..cc29876447 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -16,6 +16,8 @@ package msggateway import ( "context" + "sync/atomic" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/open-im-server/v3/pkg/common/startrpc" @@ -30,7 +32,6 @@ import ( "github.com/openimsdk/tools/mq/memamq" "github.com/openimsdk/tools/utils/datautil" "google.golang.org/grpc" - "sync/atomic" ) func (s *Server) InitServer(ctx context.Context, config *Config, disCov discovery.SvcDiscoveryRegistry, server *grpc.Server) error { @@ -111,15 +112,14 @@ func (s *Server) GetUsersOnlineStatus( } ps := new(msggateway.GetUsersOnlineStatusResp_SuccessDetail) - ps.Platform = constant.PlatformIDToName(client.PlatformID) - ps.Status = constant.OnlineStatus + ps.PlatformID = int32(client.PlatformID) ps.ConnID = client.ctx.GetConnID() ps.Token = client.token ps.IsBackground = client.IsBackground - uresp.Status = constant.OnlineStatus + uresp.Status = constant.Online uresp.DetailPlatformStatus = append(uresp.DetailPlatformStatus, ps) } - if uresp.Status == constant.OnlineStatus { + if uresp.Status == constant.Online { resp.SuccessResult = append(resp.SuccessResult, uresp) } } diff --git a/internal/msggateway/online.go b/internal/msggateway/online.go index 27b4544aaa..f29869b6e5 100644 --- a/internal/msggateway/online.go +++ b/internal/msggateway/online.go @@ -90,6 +90,19 @@ func (ws *WsServer) ChangeOnlineStatus(concurrent int) { if _, err := ws.userClient.Client.SetUserOnlineStatus(ctx, req); err != nil { log.ZError(ctx, "update user online status", err) } + for _, ss := range req.Status { + for _, online := range ss.Online { + client, _, _ := ws.clients.Get(ss.UserID, int(online)) + back := false + if len(client) > 0 { + back = client[0].IsBackground + } + ws.webhookAfterUserOnline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOnline, ss.UserID, int(online), back, ss.ConnID) + } + for _, offline := range ss.Offline { + ws.webhookAfterUserOffline(ctx, &ws.msgGatewayConfig.WebhooksConfig.AfterUserOffline, ss.UserID, int(offline), ss.ConnID) + } + } } for i := 0; i < concurrent; i++ { diff --git a/internal/push/a_test.go b/internal/push/a_test.go deleted file mode 100644 index 8b2d864071..0000000000 --- a/internal/push/a_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package push - -import ( - "github.com/openimsdk/protocol/sdkws" - "testing" -) - -func TestName(t *testing.T) { - var c ConsumerHandler - c.readCh = make(chan *sdkws.MarkAsReadTips) - - go c.loopRead() - - go func() { - for i := 0; ; i++ { - seq := int64(i + 1) - if seq%3 == 0 { - seq = 1 - } - c.readCh <- &sdkws.MarkAsReadTips{ - ConversationID: "c100", - MarkAsReadUserID: "u100", - HasReadSeq: seq, - } - } - }() - - select {} -} diff --git a/internal/rpc/auth/auth.go b/internal/rpc/auth/auth.go index d870a6c58f..06ae89d971 100644 --- a/internal/rpc/auth/auth.go +++ b/internal/rpc/auth/auth.go @@ -20,6 +20,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/config" redis2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/utils/datautil" "github.com/redis/go-redis/v9" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -71,18 +72,26 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg return nil } -func (s *authServer) UserToken(ctx context.Context, req *pbauth.UserTokenReq) (*pbauth.UserTokenResp, error) { - resp := pbauth.UserTokenResp{} +func (s *authServer) GetAdminToken(ctx context.Context, req *pbauth.GetAdminTokenReq) (*pbauth.GetAdminTokenResp, error) { + resp := pbauth.GetAdminTokenResp{} if req.Secret != s.config.Share.Secret { return nil, errs.ErrNoPermission.WrapMsg("secret invalid") } + + if !datautil.Contain(req.UserID, s.config.Share.IMAdminUserID...) { + return nil, errs.ErrArgs.WrapMsg("userID is error.", "userID", req.UserID, "adminUserID", s.config.Share.IMAdminUserID) + + } + if _, err := s.userRpcClient.GetUserInfo(ctx, req.UserID); err != nil { return nil, err } - token, err := s.authDatabase.CreateToken(ctx, req.UserID, int(req.PlatformID)) + + token, err := s.authDatabase.CreateToken(ctx, req.UserID, int(constant.AdminPlatformID)) if err != nil { return nil, err } + prommetrics.UserLoginCounter.Inc() resp.Token = token resp.ExpireTimeSeconds = s.config.RpcConfig.TokenPolicy.Expire * 24 * 60 * 60 @@ -93,6 +102,11 @@ func (s *authServer) GetUserToken(ctx context.Context, req *pbauth.GetUserTokenR if err := authverify.CheckAdmin(ctx, s.config.Share.IMAdminUserID); err != nil { return nil, err } + + if req.PlatformID == constant.AdminPlatformID { + return nil, errs.ErrNoPermission.WrapMsg("platformID invalid. platformID must not be adminPlatformID") + } + resp := pbauth.GetUserTokenResp{} if authverify.IsManagerUserID(req.UserID, s.config.Share.IMAdminUserID) { diff --git a/internal/rpc/group/callback.go b/internal/rpc/group/callback.go index e748c66dc5..559d64ff47 100644 --- a/internal/rpc/group/callback.go +++ b/internal/rpc/group/callback.go @@ -374,7 +374,7 @@ func (s *groupServer) webhookBeforeSetGroupInfoEx(ctx context.Context, before *c if req.Ex != nil { cbReq.Ex = req.Ex } - log.ZDebug(ctx, "debug CallbackBeforeSetGroupInfoEX", "ex", cbReq.Ex) + log.ZDebug(ctx, "debug CallbackBeforeSetGroupInfoEx", "ex", cbReq.Ex) if req.NeedVerification != nil { cbReq.NeedVerification = req.NeedVerification diff --git a/internal/rpc/group/db_map.go b/internal/rpc/group/db_map.go index 2f6791d5e0..26c9a46255 100644 --- a/internal/rpc/group/db_map.go +++ b/internal/rpc/group/db_map.go @@ -58,8 +58,12 @@ func UpdateGroupInfoMap(ctx context.Context, group *sdkws.GroupInfoForSet) map[s func UpdateGroupInfoExMap(ctx context.Context, group *pbgroup.SetGroupInfoExReq) (map[string]any, error) { m := make(map[string]any) - if group.GroupName != nil && group.GroupName.Value != "" { - return nil, errs.ErrArgs.WrapMsg("group name is empty") + if group.GroupName != nil { + if group.GroupName.Value != "" { + m["group_name"] = group.GroupName.Value + } else { + return nil, errs.ErrArgs.WrapMsg("group name is empty") + } } if group.Notification != nil { m["notification"] = group.Notification.Value diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index 8cb098ddeb..fce33de6fd 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1485,9 +1485,6 @@ func (g *groupServer) SetGroupMemberInfo(ctx context.Context, req *pbgroup.SetGr return nil, errs.ErrNoPermission.WrapMsg("no op user id") } isAppManagerUid := authverify.IsAppManagerUid(ctx, g.config.Share.IMAdminUserID) - for i := range req.Members { - req.Members[i].FaceURL = nil - } groupMembers := make(map[string][]*pbgroup.SetGroupMemberInfo) for i, member := range req.Members { if member.RoleLevel != nil { @@ -1764,6 +1761,7 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req } adminIDs = append(adminIDs, owners[0].UserID) + adminIDs = append(adminIDs, g.config.Share.IMAdminUserID...) if !datautil.Contain(req.UserID, adminIDs...) { return nil, errs.ErrNoPermission.WrapMsg("opUser no permission") diff --git a/internal/rpc/msg/callback.go b/internal/rpc/msg/callback.go index 3b76c25537..c66dd6ca91 100644 --- a/internal/rpc/msg/callback.go +++ b/internal/rpc/msg/callback.go @@ -67,6 +67,9 @@ func (m *msgServer) webhookBeforeSendSingleMsg(ctx context.Context, before *conf if msg.MsgData.ContentType == constant.Typing { return nil } + if !filterBeforeMsg(msg, before) { + return nil + } cbReq := &cbapi.CallbackBeforeSendSingleMsgReq{ CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackBeforeSendSingleMsgCommand), RecvID: msg.MsgData.RecvID, @@ -84,9 +87,7 @@ func (m *msgServer) webhookAfterSendSingleMsg(ctx context.Context, after *config if msg.MsgData.ContentType == constant.Typing { return } - // According to the attentionIds configuration, only some users are sent - attentionIds := after.AttentionIds - if attentionIds != nil && !datautil.Contain(msg.MsgData.RecvID, attentionIds...) && !datautil.Contain(msg.MsgData.SendID, attentionIds...) { + if !filterAfterMsg(msg, after) { return } cbReq := &cbapi.CallbackAfterSendSingleMsgReq{ @@ -98,6 +99,9 @@ func (m *msgServer) webhookAfterSendSingleMsg(ctx context.Context, after *config func (m *msgServer) webhookBeforeSendGroupMsg(ctx context.Context, before *config.BeforeConfig, msg *pbchat.SendMsgReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { + if !filterBeforeMsg(msg, before) { + return nil + } if msg.MsgData.ContentType == constant.Typing { return nil } @@ -117,6 +121,9 @@ func (m *msgServer) webhookAfterSendGroupMsg(ctx context.Context, after *config. if msg.MsgData.ContentType == constant.Typing { return } + if !filterAfterMsg(msg, after) { + return + } cbReq := &cbapi.CallbackAfterSendGroupMsgReq{ CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackAfterSendGroupMsgCommand), GroupID: msg.MsgData.GroupID, @@ -129,6 +136,9 @@ func (m *msgServer) webhookBeforeMsgModify(ctx context.Context, before *config.B if msg.MsgData.ContentType != constant.Text { return nil } + if !filterBeforeMsg(msg, before) { + return nil + } cbReq := &cbapi.CallbackMsgModifyCommandReq{ CommonCallbackReq: toCommonCallback(ctx, msg, cbapi.CallbackBeforeMsgModifyCommand), } diff --git a/internal/rpc/msg/filter.go b/internal/rpc/msg/filter.go new file mode 100644 index 0000000000..ed1a488f12 --- /dev/null +++ b/internal/rpc/msg/filter.go @@ -0,0 +1,67 @@ +package msg + +import ( + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + pbchat "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/utils/datautil" + "strconv" + "strings" +) + +const ( + separator = "-" +) + +func filterAfterMsg(msg *pbchat.SendMsgReq, after *config.AfterConfig) bool { + return filterMsg(msg, after.AttentionIds, after.AllowedTypes, after.DeniedTypes) +} + +func filterBeforeMsg(msg *pbchat.SendMsgReq, before *config.BeforeConfig) bool { + return filterMsg(msg, nil, before.AllowedTypes, before.DeniedTypes) +} + +func filterMsg(msg *pbchat.SendMsgReq, attentionIds, allowedTypes, deniedTypes []string) bool { + // According to the attentionIds configuration, only some users are sent + if len(attentionIds) != 0 && !datautil.Contains([]string{msg.MsgData.SendID, msg.MsgData.RecvID}, attentionIds...) { + return false + } + if len(allowedTypes) != 0 && !isInInterval(msg.MsgData.ContentType, allowedTypes) { + return false + } + if len(deniedTypes) != 0 && isInInterval(msg.MsgData.ContentType, deniedTypes) { + return false + } + return true +} + +func isInInterval(contentType int32, interval []string) bool { + for _, v := range interval { + if strings.Contains(v, separator) { + // is interval + bounds := strings.Split(v, separator) + if len(bounds) != 2 { + continue + } + bottom, err := strconv.Atoi(bounds[0]) + if err != nil { + continue + } + top, err := strconv.Atoi(bounds[1]) + if err != nil { + continue + } + if datautil.BetweenEq(int(contentType), bottom, top) { + return true + } + } else { + iv, err := strconv.Atoi(v) + if err != nil { + continue + } + if int(contentType) == iv { + return true + } + } + } + return false +} diff --git a/internal/rpc/relation/callback.go b/internal/rpc/relation/callback.go index 69c4c9e0ec..09debdea18 100644 --- a/internal/rpc/relation/callback.go +++ b/internal/rpc/relation/callback.go @@ -138,6 +138,18 @@ func (s *friendServer) webhookBeforeAddFriendAgree(ctx context.Context, before * }) } +func (s *friendServer) webhookAfterAddFriendAgree(ctx context.Context, after *config.AfterConfig, req *relation.RespondFriendApplyReq) { + cbReq := &cbapi.CallbackAfterAddFriendAgreeReq{ + CallbackCommand: cbapi.CallbackAfterAddFriendAgreeCommand, + FromUserID: req.FromUserID, + ToUserID: req.ToUserID, + HandleMsg: req.HandleMsg, + HandleResult: req.HandleResult, + } + resp := &cbapi.CallbackAfterAddFriendAgreeResp{} + s.webhookClient.AsyncPost(ctx, cbReq.GetCallbackCommand(), cbReq, resp, after) +} + func (s *friendServer) webhookBeforeImportFriends(ctx context.Context, before *config.BeforeConfig, req *relation.ImportFriendReq) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { cbReq := &cbapi.CallbackBeforeImportFriendsReq{ diff --git a/internal/rpc/relation/friend.go b/internal/rpc/relation/friend.go index f049420d93..2f4843a8ef 100644 --- a/internal/rpc/relation/friend.go +++ b/internal/rpc/relation/friend.go @@ -212,6 +212,7 @@ func (s *friendServer) RespondFriendApply(ctx context.Context, req *relation.Res if err != nil { return nil, err } + s.webhookAfterAddFriendAgree(ctx, &s.config.WebhooksConfig.AfterAddFriendAgree, req) s.notificationSender.FriendApplicationAgreedNotification(ctx, req) return resp, nil } @@ -272,7 +273,14 @@ func (s *friendServer) SetFriendRemark(ctx context.Context, req *relation.SetFri return &relation.SetFriendRemarkResp{}, nil } -// ok. +func (s *friendServer) GetFriendInfo(ctx context.Context, req *relation.GetFriendInfoReq) (*relation.GetFriendInfoResp, error) { + friends, err := s.db.FindFriendsWithError(ctx, req.OwnerUserID, req.FriendUserIDs) + if err != nil { + return nil, err + } + return &relation.GetFriendInfoResp{FriendInfos: convert.FriendOnlyDB2PbOnly(friends)}, nil +} + func (s *friendServer) GetDesignatedFriends(ctx context.Context, req *relation.GetDesignatedFriendsReq) (resp *relation.GetDesignatedFriendsResp, err error) { resp = &relation.GetDesignatedFriendsResp{} if datautil.Duplicate(req.FriendUserIDs) { diff --git a/internal/rpc/user/online.go b/internal/rpc/user/online.go index 4e7823306f..0e5365ed93 100644 --- a/internal/rpc/user/online.go +++ b/internal/rpc/user/online.go @@ -2,6 +2,7 @@ package user import ( "context" + "github.com/openimsdk/tools/utils/datautil" "github.com/openimsdk/protocol/constant" @@ -61,7 +62,7 @@ func (s *userServer) SetUserStatus(ctx context.Context, req *pbuser.SetUserStatu case constant.Online: online = []int32{req.PlatformID} case constant.Offline: - online = []int32{req.PlatformID} + offline = []int32{req.PlatformID} } if err := s.online.SetUserOnline(ctx, req.UserID, online, offline); err != nil { return nil, err diff --git a/pkg/callbackstruct/constant.go b/pkg/callbackstruct/constant.go index 5d136d3da3..73f89a7193 100644 --- a/pkg/callbackstruct/constant.go +++ b/pkg/callbackstruct/constant.go @@ -25,6 +25,7 @@ const ( CallbackBeforeAddBlackCommand = "callbackBeforeAddBlackCommand" CallbackAfterAddFriendCommand = "callbackAfterAddFriendCommand" CallbackBeforeAddFriendAgreeCommand = "callbackBeforeAddFriendAgreeCommand" + CallbackAfterAddFriendAgreeCommand = "callbackAfterAddFriendAgreeCommand" CallbackAfterDeleteFriendCommand = "callbackAfterDeleteFriendCommand" CallbackBeforeImportFriendsCommand = "callbackBeforeImportFriendsCommand" CallbackAfterImportFriendsCommand = "callbackAfterImportFriendsCommand" diff --git a/pkg/callbackstruct/friend.go b/pkg/callbackstruct/friend.go index 3674a34dab..a81746bfdd 100644 --- a/pkg/callbackstruct/friend.go +++ b/pkg/callbackstruct/friend.go @@ -90,6 +90,18 @@ type CallbackBeforeAddFriendAgreeResp struct { CommonCallbackResp } +type CallbackAfterAddFriendAgreeReq struct { + CallbackCommand `json:"callbackCommand"` + FromUserID string `json:"fromUserID" ` + ToUserID string `json:"blackUserID"` + HandleResult int32 `json:"HandleResult"` + HandleMsg string `json:"HandleMsg"` +} + +type CallbackAfterAddFriendAgreeResp struct { + CommonCallbackResp +} + type CallbackAfterDeleteFriendReq struct { CallbackCommand `json:"callbackCommand"` OwnerUserID string `json:"ownerUserID" ` diff --git a/pkg/common/cmd/msg_gateway_test.go b/pkg/common/cmd/msg_gateway_test.go index 2b68a3e3ab..929abcd863 100644 --- a/pkg/common/cmd/msg_gateway_test.go +++ b/pkg/common/cmd/msg_gateway_test.go @@ -15,13 +15,14 @@ package cmd import ( + "math" + "testing" + "github.com/openimsdk/protocol/auth" "github.com/openimsdk/tools/apiresp" "github.com/openimsdk/tools/utils/jsonutil" "github.com/stretchr/testify/mock" "go.mongodb.org/mongo-driver/bson/primitive" - "math" - "testing" ) // MockRootCmd is a mock type for the RootCmd type @@ -39,7 +40,7 @@ func TestName(t *testing.T) { ErrCode: 1234, ErrMsg: "test", ErrDlt: "4567", - Data: &auth.UserTokenResp{ + Data: &auth.GetUserTokenResp{ Token: "1234567", ExpireTimeSeconds: math.MaxInt64, }, @@ -51,7 +52,7 @@ func TestName(t *testing.T) { t.Log(string(data)) var rReso apiresp.ApiResponse - rReso.Data = &auth.UserTokenResp{} + rReso.Data = &auth.GetUserTokenResp{} if err := jsonutil.JsonUnmarshal(data, &rReso); err != nil { panic(err) diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 830b1ef9d0..77fcbb8aa1 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -345,15 +345,19 @@ type Redis struct { } type BeforeConfig struct { - Enable bool `mapstructure:"enable"` - Timeout int `mapstructure:"timeout"` - FailedContinue bool `mapstructure:"failedContinue"` + Enable bool `mapstructure:"enable"` + Timeout int `mapstructure:"timeout"` + FailedContinue bool `mapstructure:"failedContinue"` + AllowedTypes []string `mapstructure:"allowedTypes"` + DeniedTypes []string `mapstructure:"deniedTypes"` } type AfterConfig struct { Enable bool `mapstructure:"enable"` Timeout int `mapstructure:"timeout"` AttentionIds []string `mapstructure:"attentionIds"` + AllowedTypes []string `mapstructure:"allowedTypes"` + DeniedTypes []string `mapstructure:"deniedTypes"` } type Share struct { @@ -434,6 +438,7 @@ type Webhooks struct { BeforeAddBlack BeforeConfig `mapstructure:"beforeAddBlack"` AfterAddFriend AfterConfig `mapstructure:"afterAddFriend"` BeforeAddFriendAgree BeforeConfig `mapstructure:"beforeAddFriendAgree"` + AfterAddFriendAgree AfterConfig `mapstructure:"afterAddFriendAgree"` AfterDeleteFriend AfterConfig `mapstructure:"afterDeleteFriend"` BeforeImportFriends BeforeConfig `mapstructure:"beforeImportFriends"` AfterImportFriends AfterConfig `mapstructure:"afterImportFriends"` diff --git a/pkg/common/convert/friend.go b/pkg/common/convert/friend.go index 8d6cfad183..6d346b0f4a 100644 --- a/pkg/common/convert/friend.go +++ b/pkg/common/convert/friend.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/protocol/relation" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/utils/datautil" @@ -35,9 +36,7 @@ func FriendPb2DB(friend *sdkws.FriendInfo) *model.Friend { return dbFriend } -func FriendDB2Pb(ctx context.Context, friendDB *model.Friend, - getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error), -) (*sdkws.FriendInfo, error) { +func FriendDB2Pb(ctx context.Context, friendDB *model.Friend, getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error)) (*sdkws.FriendInfo, error) { users, err := getUsers(ctx, []string{friendDB.FriendUserID}) if err != nil { return nil, err @@ -53,11 +52,7 @@ func FriendDB2Pb(ctx context.Context, friendDB *model.Friend, }, nil } -func FriendsDB2Pb( - ctx context.Context, - friendsDB []*model.Friend, - getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error), -) (friendsPb []*sdkws.FriendInfo, err error) { +func FriendsDB2Pb(ctx context.Context, friendsDB []*model.Friend, getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error)) (friendsPb []*sdkws.FriendInfo, err error) { if len(friendsDB) == 0 { return nil, nil } @@ -86,7 +81,21 @@ func FriendsDB2Pb( friendsPb = append(friendsPb, friendPb) } return friendsPb, nil +} +func FriendOnlyDB2PbOnly(friendsDB []*model.Friend) []*relation.FriendInfoOnly { + return datautil.Slice(friendsDB, func(f *model.Friend) *relation.FriendInfoOnly { + return &relation.FriendInfoOnly{ + OwnerUserID: f.OwnerUserID, + FriendUserID: f.FriendUserID, + Remark: f.Remark, + CreateTime: f.CreateTime.UnixMilli(), + AddSource: f.AddSource, + OperatorUserID: f.OperatorUserID, + Ex: f.Ex, + IsPinned: f.IsPinned, + } + }) } func FriendRequestDB2Pb(ctx context.Context, friendRequests []*model.FriendRequest, getUsers func(ctx context.Context, userIDs []string) (map[string]*sdkws.UserInfo, error)) ([]*sdkws.FriendRequest, error) { diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index cb06a197df..94f18b3ae3 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -16,6 +16,7 @@ package controller import ( "context" + "github.com/openimsdk/tools/log" "github.com/golang-jwt/jwt/v4" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -77,12 +78,23 @@ func (a *authDatabase) CreateToken(ctx context.Context, userID string, platformI return "", err } } + + const adminTokenMaxNum = 30 + if platformID == constant.AdminPlatformID { + if len(kickedTokenKey) > adminTokenMaxNum { + kickedTokenKey = kickedTokenKey[:len(kickedTokenKey)-adminTokenMaxNum] + } else { + kickedTokenKey = nil + } + } + if len(kickedTokenKey) != 0 { for _, k := range kickedTokenKey { err := a.cache.SetTokenFlagEx(ctx, userID, platformID, k, constant.KickedToken) if err != nil { return "", err } + log.ZDebug(ctx, "kicked token in create token", "token", k) } } diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index 5e540a2c33..c5dfd011f5 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -254,7 +254,7 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver func (db *msgTransferDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { for userID, seq := range userSeqMap { - if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { + if err := db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { return err } } diff --git a/pkg/msgprocessor/conversation.go b/pkg/msgprocessor/conversation.go index f8140cc7df..04d772d161 100644 --- a/pkg/msgprocessor/conversation.go +++ b/pkg/msgprocessor/conversation.go @@ -39,7 +39,9 @@ func GetNotificationConversationIDByMsg(msg *sdkws.MsgData) string { case constant.ReadGroupChatType: return "n_" + msg.GroupID case constant.NotificationChatType: - return "n_" + msg.SendID + "_" + msg.RecvID + l := []string{msg.SendID, msg.RecvID} + sort.Strings(l) + return "n_" + strings.Join(l, "_") } return "" } @@ -55,21 +57,11 @@ func GetChatConversationIDByMsg(msg *sdkws.MsgData) string { case constant.ReadGroupChatType: return "sg_" + msg.GroupID case constant.NotificationChatType: - return "sn_" + msg.SendID + "_" + msg.RecvID - } - - return "" -} - -func GenConversationUniqueKey(msg *sdkws.MsgData) string { - switch msg.SessionType { - case constant.SingleChatType, constant.NotificationChatType: l := []string{msg.SendID, msg.RecvID} sort.Strings(l) - return strings.Join(l, "_") - case constant.ReadGroupChatType: - return msg.GroupID + return "sn_" + strings.Join(l, "_") } + return "" } @@ -94,10 +86,12 @@ func GetConversationIDByMsg(msg *sdkws.MsgData) string { } return "sg_" + msg.GroupID // super group chat case constant.NotificationChatType: + l := []string{msg.SendID, msg.RecvID} + sort.Strings(l) if !options.IsNotNotification() { - return "n_" + msg.SendID + "_" + msg.RecvID // super group chat + return "n_" + strings.Join(l, "_") } - return "sn_" + msg.SendID + "_" + msg.RecvID // server notification chat + return "sn_" + strings.Join(l, "_") } return "" } @@ -120,30 +114,6 @@ func GetConversationIDBySessionType(sessionType int, ids ...string) string { return "" } -func GetNotificationConversationIDByConversationID(conversationID string) string { - l := strings.Split(conversationID, "_") - if len(l) > 1 { - l[0] = "n" - return strings.Join(l, "_") - } - - return "" -} - -func GetNotificationConversationID(sessionType int, ids ...string) string { - sort.Strings(ids) - if len(ids) > 2 || len(ids) < 1 { - return "" - } - switch sessionType { - case constant.SingleChatType: - return "n_" + strings.Join(ids, "_") // single chat - case constant.ReadGroupChatType: - return "n_" + ids[0] // super group chat - } - return "" -} - func IsNotification(conversationID string) bool { return strings.HasPrefix(conversationID, "n_") } @@ -152,30 +122,6 @@ func IsNotificationByMsg(msg *sdkws.MsgData) bool { return !Options(msg.Options).IsNotNotification() } -func ParseConversationID(msg *sdkws.MsgData) (isNotification bool, conversationID string) { - options := Options(msg.Options) - switch msg.SessionType { - case constant.SingleChatType: - l := []string{msg.SendID, msg.RecvID} - sort.Strings(l) - if !options.IsNotNotification() { - return true, "n_" + strings.Join(l, "_") - } - return false, "si_" + strings.Join(l, "_") // single chat - case constant.ReadGroupChatType: - if !options.IsNotNotification() { - return true, "n_" + msg.GroupID // super group chat - } - return false, "sg_" + msg.GroupID // super group chat - case constant.NotificationChatType: - if !options.IsNotNotification() { - return true, "n_" + msg.SendID + "_" + msg.RecvID // super group chat - } - return false, "sn_" + msg.SendID + "_" + msg.RecvID // server notification chat - } - return false, "" -} - type MsgBySeq []*sdkws.MsgData func (s MsgBySeq) Len() int { diff --git a/pkg/msgprocessor/conversation_test.go b/pkg/msgprocessor/conversation_test.go deleted file mode 100644 index 32601baec4..0000000000 --- a/pkg/msgprocessor/conversation_test.go +++ /dev/null @@ -1,334 +0,0 @@ -// Copyright © 2023 OpenIM. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package msgprocessor - -import ( - "testing" - - "github.com/openimsdk/protocol/sdkws" - "google.golang.org/protobuf/proto" -) - -func TestGetNotificationConversationIDByMsg(t *testing.T) { - type args struct { - msg *sdkws.MsgData - } - tests := []struct { - name string - args args - want string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GetNotificationConversationIDByMsg(tt.args.msg); got != tt.want { - t.Errorf("GetNotificationConversationIDByMsg() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetChatConversationIDByMsg(t *testing.T) { - type args struct { - msg *sdkws.MsgData - } - tests := []struct { - name string - args args - want string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GetChatConversationIDByMsg(tt.args.msg); got != tt.want { - t.Errorf("GetChatConversationIDByMsg() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGenConversationUniqueKey(t *testing.T) { - type args struct { - msg *sdkws.MsgData - } - tests := []struct { - name string - args args - want string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GenConversationUniqueKey(tt.args.msg); got != tt.want { - t.Errorf("GenConversationUniqueKey() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetConversationIDByMsg(t *testing.T) { - type args struct { - msg *sdkws.MsgData - } - tests := []struct { - name string - args args - want string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GetConversationIDByMsg(tt.args.msg); got != tt.want { - t.Errorf("GetConversationIDByMsg() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetConversationIDBySessionType(t *testing.T) { - type args struct { - sessionType int - ids []string - } - tests := []struct { - name string - args args - want string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GetConversationIDBySessionType(tt.args.sessionType, tt.args.ids...); got != tt.want { - t.Errorf("GetConversationIDBySessionType() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetNotificationConversationIDByConversationID(t *testing.T) { - type args struct { - conversationID string - } - tests := []struct { - name string - args args - want string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GetNotificationConversationIDByConversationID(tt.args.conversationID); got != tt.want { - t.Errorf("GetNotificationConversationIDByConversationID() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestGetNotificationConversationID(t *testing.T) { - type args struct { - sessionType int - ids []string - } - tests := []struct { - name string - args args - want string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := GetNotificationConversationID(tt.args.sessionType, tt.args.ids...); got != tt.want { - t.Errorf("GetNotificationConversationID() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestIsNotification(t *testing.T) { - type args struct { - conversationID string - } - tests := []struct { - name string - args args - want bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := IsNotification(tt.args.conversationID); got != tt.want { - t.Errorf("IsNotification() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestIsNotificationByMsg(t *testing.T) { - type args struct { - msg *sdkws.MsgData - } - tests := []struct { - name string - args args - want bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := IsNotificationByMsg(tt.args.msg); got != tt.want { - t.Errorf("IsNotificationByMsg() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParseConversationID(t *testing.T) { - type args struct { - msg *sdkws.MsgData - } - tests := []struct { - name string - args args - wantIsNotification bool - wantConversationID string - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotIsNotification, gotConversationID := ParseConversationID(tt.args.msg) - if gotIsNotification != tt.wantIsNotification { - t.Errorf("ParseConversationID() gotIsNotification = %v, want %v", gotIsNotification, tt.wantIsNotification) - } - if gotConversationID != tt.wantConversationID { - t.Errorf("ParseConversationID() gotConversationID = %v, want %v", gotConversationID, tt.wantConversationID) - } - }) - } -} - -func TestMsgBySeq_Len(t *testing.T) { - tests := []struct { - name string - s MsgBySeq - want int - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.s.Len(); got != tt.want { - t.Errorf("MsgBySeq.Len() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestMsgBySeq_Less(t *testing.T) { - type args struct { - i int - j int - } - tests := []struct { - name string - s MsgBySeq - args args - want bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.s.Less(tt.args.i, tt.args.j); got != tt.want { - t.Errorf("MsgBySeq.Less() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestMsgBySeq_Swap(t *testing.T) { - type args struct { - i int - j int - } - tests := []struct { - name string - s MsgBySeq - args args - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.s.Swap(tt.args.i, tt.args.j) - }) - } -} - -func TestPb2String(t *testing.T) { - type args struct { - pb proto.Message - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := Pb2String(tt.args.pb) - if (err != nil) != tt.wantErr { - t.Errorf("Pb2String() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("Pb2String() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestString2Pb(t *testing.T) { - type args struct { - s string - pb proto.Message - } - tests := []struct { - name string - args args - wantErr bool - }{ - // TODO: Add test cases. - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := String2Pb(tt.args.s, tt.args.pb); (err != nil) != tt.wantErr { - t.Errorf("String2Pb() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/tools/url2im/pkg/api.go b/tools/url2im/pkg/api.go index 5bf48c4ea5..0ef8c1db77 100644 --- a/tools/url2im/pkg/api.go +++ b/tools/url2im/pkg/api.go @@ -23,7 +23,6 @@ import ( "net/http" "github.com/openimsdk/protocol/auth" - "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" ) @@ -88,14 +87,13 @@ func (a *Api) apiPost(ctx context.Context, path string, req any, resp any) error return nil } -func (a *Api) GetToken(ctx context.Context) (string, error) { - req := auth.UserTokenReq{ - UserID: a.UserID, - Secret: a.Secret, - PlatformID: constant.AdminPlatformID, - } - var resp auth.UserTokenResp - if err := a.apiPost(ctx, "/auth/user_token", &req, &resp); err != nil { +func (a *Api) GetAdminToken(ctx context.Context) (string, error) { + req := auth.GetAdminTokenReq{ + UserID: a.UserID, + Secret: a.Secret, + } + var resp auth.GetAdminTokenResp + if err := a.apiPost(ctx, "/auth/get_admin_token", &req, &resp); err != nil { return "", err } return resp.Token, nil diff --git a/tools/url2im/pkg/manage.go b/tools/url2im/pkg/manage.go index 3664baa25b..9dc1de8010 100644 --- a/tools/url2im/pkg/manage.go +++ b/tools/url2im/pkg/manage.go @@ -21,7 +21,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/openimsdk/tools/errs" "io" "log" "net/http" @@ -34,6 +33,8 @@ import ( "sync/atomic" "time" + "github.com/openimsdk/tools/errs" + "github.com/openimsdk/protocol/third" ) @@ -95,7 +96,7 @@ func (m *Manage) Run() error { } var err error ctx := context.WithValue(m.ctx, "operationID", fmt.Sprintf("%s_init", m.prefix)) - m.api.Token, err = m.api.GetToken(ctx) + m.api.Token, err = m.api.GetAdminToken(ctx) if err != nil { return err } diff --git a/version/version b/version/version index 0be1fc7d24..aaaff91926 100644 --- a/version/version +++ b/version/version @@ -1 +1 @@ -3.8.0 \ No newline at end of file +3.8.1 \ No newline at end of file From 646f1b9194dce2b24ca19b339118ec9fdb9bf164 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 25 Oct 2024 17:33:49 +0800 Subject: [PATCH 08/12] merge --- pkg/common/storage/controller/auth.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/common/storage/controller/auth.go b/pkg/common/storage/controller/auth.go index b6ffcca15c..6f783cbe90 100644 --- a/pkg/common/storage/controller/auth.go +++ b/pkg/common/storage/controller/auth.go @@ -9,6 +9,7 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" "github.com/openimsdk/protocol/constant" "github.com/openimsdk/tools/errs" + "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/tokenverify" ) From ee04c156b588b03313fa3fdac90b3195148aa364 Mon Sep 17 00:00:00 2001 From: OpenIM-Gordon <46924906+FGadvancer@users.noreply.github.com> Date: Thu, 31 Oct 2024 18:32:03 +0800 Subject: [PATCH 09/12] merge: update code from main to v3.8-js-sdk-only. (#2818) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: implement merge milestone PR to target-branch. (#2796) * build: improve workflows logic. (#2801) * fix: improve time condition check mehtod. (#2804) * fix: improve time condition check mehtod. * fix * fix: webhook before online push (#2805) * fix: set own read seq in MongoDB when sender send a message. (#2808) * fix: solve err Notification when setGroupInfo. (#2806) * fix: solve err Notification when setGroupInfo. * build: update checkout version. * fix: update notification contents. * Introducing OpenIM Guru on Gurubase.io (#2788) * feat: support app update service (#2811) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: add ApplicationVersion * feat: add ApplicationVersion --------- Co-authored-by: withchao * feat: ApplicationVersion move chat (#2813) * fix: GroupApplicationAcceptedNotification * fix: GroupApplicationAcceptedNotification * fix: NotificationUserInfoUpdate * cicd: robot automated Change * fix: component * fix: getConversationInfo * feat: cron task * feat: cron task * feat: cron task * feat: cron task * feat: cron task * fix: minio config url recognition error * update gomake version * update gomake version * fix: seq conversion bug * fix: redis pipe exec * fix: ImportFriends * fix: A large number of logs keysAndValues ​​length is not even * feat: mark read aggregate write * feat: online status supports redis cluster * feat: online status supports redis cluster * feat: online status supports redis cluster * merge * merge * read seq is written to mongo * read seq is written to mongo * fix: invitation to join group notification * fix: friend op_user_id * feat: optimizing asynchronous context * feat: optimizing memamq size * feat: add GetSeqMessage * feat: GroupApplicationAgreeMemberEnterNotification * feat: GroupApplicationAgreeMemberEnterNotification * feat: go.mod * feat: go.mod * feat: join group notification and get seq * feat: join group notification and get seq * feat: avoid pulling messages from sessions with a large number of max seq values of 0 * feat: API supports gzip * go.mod * fix: nil pointer error on close * fix: listen error * fix: listen error * update go.mod * feat: add log * fix: token parse token value * fix: GetMsgBySeqs boundary issues * fix: sn_ not sort * fix: sn_ not sort * fix: sn_ not sort * fix: jssdk add * fix: jssdk support * fix: jssdk support * fix: jssdk support * fix: the message I sent is not set to read seq in mongodb * fix: cannot modify group member avatars * fix: MemberEnterNotification * fix: MemberEnterNotification * fix: MsgData status * feat: add ApplicationVersion * feat: ApplicationVersion move chat --------- Co-authored-by: withchao * fix: improve condition check. (#2815) --------- Co-authored-by: Monet Lee Co-authored-by: icey-yu <119291641+icey-yu@users.noreply.github.com> Co-authored-by: Kürşat Aktaş Co-authored-by: chao <48119764+withchao@users.noreply.github.com> Co-authored-by: withchao --- .../cleanup-after-milestone-prs-merged.yml | 65 ++++++ .github/workflows/go-build-test.yml | 4 - .github/workflows/merge-from-milestone.yml | 218 ++++++++++++++++++ README.md | 1 + config/notification.yml | 23 +- go.mod | 2 +- go.sum | 4 +- internal/api/router.go | 9 - internal/api/third.go | 20 -- internal/msgtransfer/init.go | 3 + .../msgtransfer/online_history_msg_handler.go | 99 +++++--- internal/push/callback.go | 3 +- internal/rpc/group/group.go | 11 +- internal/rpc/third/application.go | 117 ---------- internal/rpc/third/third.go | 30 +-- pkg/common/storage/cache/application.go | 11 - .../storage/cache/cachekey/application.go | 9 - pkg/common/storage/cache/redis/application.go | 43 ---- pkg/common/storage/controller/application.go | 69 ------ pkg/common/storage/controller/msg_transfer.go | 35 +-- pkg/common/storage/database/application.go | 17 -- .../storage/database/mgo/application.go | 82 ------- pkg/common/storage/model/application.go | 1 + 23 files changed, 400 insertions(+), 476 deletions(-) create mode 100644 .github/workflows/cleanup-after-milestone-prs-merged.yml create mode 100644 .github/workflows/merge-from-milestone.yml delete mode 100644 internal/rpc/third/application.go delete mode 100644 pkg/common/storage/cache/application.go delete mode 100644 pkg/common/storage/cache/cachekey/application.go delete mode 100644 pkg/common/storage/cache/redis/application.go delete mode 100644 pkg/common/storage/controller/application.go delete mode 100644 pkg/common/storage/database/application.go delete mode 100644 pkg/common/storage/database/mgo/application.go diff --git a/.github/workflows/cleanup-after-milestone-prs-merged.yml b/.github/workflows/cleanup-after-milestone-prs-merged.yml new file mode 100644 index 0000000000..8a3e381d60 --- /dev/null +++ b/.github/workflows/cleanup-after-milestone-prs-merged.yml @@ -0,0 +1,65 @@ +name: Cleanup After Milestone PRs Merged + +on: + pull_request: + types: + - closed + +jobs: + handle_pr: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4.2.0 + + - name: Get the PR title and extract PR numbers + id: extract_pr_numbers + run: | + # Get the PR title + PR_TITLE="${{ github.event.pull_request.title }}" + + echo "PR Title: $PR_TITLE" + + # Extract PR numbers from the title + PR_NUMBERS=$(echo "$PR_TITLE" | grep -oE "#[0-9]+" | tr -d '#' | tr '\n' ' ') + echo "Extracted PR Numbers: $PR_NUMBERS" + + # Save PR numbers to a file + echo "$PR_NUMBERS" > pr_numbers.txt + echo "Saved PR Numbers to pr_numbers.txt" + + # Check if the title matches a specific pattern + if echo "$PR_TITLE" | grep -qE "^deps: Merge( #[0-9]+)+ PRs into .+"; then + echo "proceed=true" >> $GITHUB_OUTPUT + else + echo "proceed=false" >> $GITHUB_OUTPUT + fi + + - name: Use extracted PR numbers and label PRs + if: (steps.extract_pr_numbers.outputs.proceed == 'true' || contains(github.event.pull_request.labels.*.name, 'milestone-merge')) && github.event.pull_request.merged == true + run: | + # Read the previously saved PR numbers + PR_NUMBERS=$(cat pr_numbers.txt) + echo "Using extracted PR Numbers: $PR_NUMBERS" + + # Loop through each PR number and add label + for PR_NUMBER in $PR_NUMBERS; do + echo "Adding 'cherry-picked' label to PR #$PR_NUMBER" + curl -X POST \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/issues/$PR_NUMBER/labels \ + -d '{"labels":["cherry-picked"]}' + done + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Delete branch after PR close + if: steps.extract_pr_numbers.outputs.proceed == 'true' || contains(github.event.pull_request.labels.*.name, 'milestone-merge') + run: | + BRANCH_NAME="${{ github.event.pull_request.head.ref }}" + echo "Branch to delete: $BRANCH_NAME" + git push origin --delete "$BRANCH_NAME" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/go-build-test.yml b/.github/workflows/go-build-test.yml index 1ed8f03977..5b37bf47db 100644 --- a/.github/workflows/go-build-test.yml +++ b/.github/workflows/go-build-test.yml @@ -2,11 +2,7 @@ name: Go Build Test on: push: - branches: - - main pull_request: - branches: - - main paths-ignore: - '**/*.md' diff --git a/.github/workflows/merge-from-milestone.yml b/.github/workflows/merge-from-milestone.yml new file mode 100644 index 0000000000..44b4f81f47 --- /dev/null +++ b/.github/workflows/merge-from-milestone.yml @@ -0,0 +1,218 @@ +name: Create Pre-Release PR from Milestone + +permissions: + contents: write + pull-requests: write + issues: write + +on: + workflow_dispatch: + inputs: + milestone_name: + description: 'Milestone name to collect closed PRs from' + required: true + default: 'v3.8.2' + target_branch: + description: 'Target branch to merge the consolidated PR' + required: true + default: 'pre-release-v3.8.2' + +env: + MILESTONE_NAME: ${{ github.event.inputs.milestone_name || 'v3.8.2' }} + TARGET_BRANCH: ${{ github.event.inputs.target_branch || 'pre-release-v3.8.2' }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_TOKEN: ${{ secrets.BOT_TOKEN }} + LABEL_NAME: cherry-picked + TEMP_DIR: /tmp # Using /tmp as the temporary directory + +jobs: + cherry_pick_milestone_prs: + runs-on: ubuntu-latest + steps: + - name: Setup temp directory + run: | + # Create the temporary directory and initialize necessary files + mkdir -p ${{ env.TEMP_DIR }} + touch ${{ env.TEMP_DIR }}/pr_numbers.txt + touch ${{ env.TEMP_DIR }}/commit_hashes.txt + touch ${{ env.TEMP_DIR }}/pr_title.txt + touch ${{ env.TEMP_DIR }}/pr_body.txt + touch ${{ env.TEMP_DIR }}/created_pr_number.txt + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.BOT_TOKEN }} + + - name: Setup Git User for OpenIM-Robot + run: | + # Set up Git credentials for the bot + git config --global user.email "OpenIM-Robot@users.noreply.github.com" + git config --global user.name "OpenIM-Robot" + + - name: Fetch Milestone ID and Filter PR Numbers + env: + MILESTONE_NAME: ${{ env.MILESTONE_NAME }} + run: | + # Fetch milestone details and extract milestone ID + milestones=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/milestones") + milestone_id=$(echo "$milestones" | grep -B3 "\"title\": \"$MILESTONE_NAME\"" | grep '"number":' | head -n1 | grep -o '[0-9]\+') + if [ -z "$milestone_id" ]; then + echo "Milestone '$MILESTONE_NAME' not found. Exiting." + exit 1 + fi + echo "Milestone ID: $milestone_id" + echo "MILESTONE_ID=$milestone_id" >> $GITHUB_ENV + + # Fetch issues for the milestone + issues=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/issues?milestone=$milestone_id&state=closed&per_page=100") + + > ${{ env.TEMP_DIR }}/pr_numbers.txt + + # Filter PRs that do not have the 'cherry-picked' label + for pr_number in $(echo "$issues" | jq -r '.[] | select(.pull_request != null) | .number'); do + labels=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/issues/$pr_number/labels" | jq -r '.[].name') + + if ! echo "$labels" | grep -q "${LABEL_NAME}"; then + echo "PR #$pr_number does not have the 'cherry-picked' label. Adding to the list." + echo "$pr_number" >> ${{ env.TEMP_DIR }}/pr_numbers.txt + else + echo "PR #$pr_number already has the 'cherry-picked' label. Skipping." + fi + done + + # Sort the filtered PR numbers + sort -n ${{ env.TEMP_DIR }}/pr_numbers.txt -o ${{ env.TEMP_DIR }}/pr_numbers.txt + + echo "Filtered and sorted PR numbers:" + cat ${{ env.TEMP_DIR }}/pr_numbers.txt || echo "No closed PR numbers found for milestone." + + - name: Fetch Merge Commits for PRs and Generate Title and Body + run: | + # Ensure the files are initialized + > ${{ env.TEMP_DIR }}/commit_hashes.txt + > ${{ env.TEMP_DIR }}/pr_title.txt + > ${{ env.TEMP_DIR }}/pr_body.txt + + # Write description to the PR body + echo "### Description:" >> ${{ env.TEMP_DIR }}/pr_body.txt + echo "Merging PRs from milestone \`$MILESTONE_NAME\` into target branch \`$TARGET_BRANCH\`." >> ${{ env.TEMP_DIR }}/pr_body.txt + echo "" >> ${{ env.TEMP_DIR }}/pr_body.txt + echo "### Need Merge PRs:" >> ${{ env.TEMP_DIR }}/pr_body.txt + + pr_numbers_in_title="" + + # Process sorted PR numbers and generate commit hashes + for pr_number in $(cat ${{ env.TEMP_DIR }}/pr_numbers.txt); do + echo "Processing PR #$pr_number" + pr_details=$(curl -s -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/$pr_number") + pr_title=$(echo "$pr_details" | jq -r '.title') + merge_commit=$(echo "$pr_details" | jq -r '.merge_commit_sha') + short_commit_hash=$(echo "$merge_commit" | cut -c 1-7) + + # Append PR details to the body + echo "- $pr_title: (#$pr_number) ($short_commit_hash)" >> ${{ env.TEMP_DIR }}/pr_body.txt + + if [ "$merge_commit" != "null" ];then + echo "$merge_commit" >> ${{ env.TEMP_DIR }}/commit_hashes.txt + echo "#$pr_number" >> ${{ env.TEMP_DIR }}/pr_title.txt + pr_numbers_in_title="$pr_numbers_in_title #$pr_number" + fi + done + + commit_hashes=$(cat ${{ env.TEMP_DIR }}/commit_hashes.txt | tr '\n' ' ') + first_commit_hash=$(head -n 1 ${{ env.TEMP_DIR }}/commit_hashes.txt) + cherry_pick_branch="cherry-pick-${first_commit_hash:0:7}" + echo "COMMIT_HASHES=$commit_hashes" >> $GITHUB_ENV + echo "CHERRY_PICK_BRANCH=$cherry_pick_branch" >> $GITHUB_ENV + echo "pr_numbers_in_title=$pr_numbers_in_title" >> $GITHUB_ENV + + - name: Pull and Cherry-pick Commits, Then Push + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_TOKEN: ${{ secrets.BOT_TOKEN }} + run: | + # Fetch and pull the latest changes from the target branch + git fetch origin + git checkout $TARGET_BRANCH + git pull origin $TARGET_BRANCH + + # Create a new branch for cherry-picking + git checkout -b $CHERRY_PICK_BRANCH + + # Cherry-pick the commits and handle conflicts + for commit_hash in $COMMIT_HASHES; do + echo "Attempting to cherry-pick commit $commit_hash" + if ! git cherry-pick "$commit_hash" --strategy=recursive -X theirs; then + echo "Conflict detected for $commit_hash. Resolving with incoming changes." + conflict_files=$(git diff --name-only --diff-filter=U) + echo "Conflicting files:" + echo "$conflict_files" + + for file in $conflict_files; do + if [ -f "$file" ]; then + echo "Resolving conflict for $file" + git add "$file" + else + echo "File $file has been deleted. Skipping." + git rm "$file" + fi + done + + echo "Conflicts resolved. Continuing cherry-pick." + git cherry-pick --continue + else + echo "Cherry-pick successful for commit $commit_hash." + fi + done + + # Push the cherry-pick branch to the repository + git remote set-url origin "https://${BOT_TOKEN}@github.com/${{ github.repository }}.git" + git push origin $CHERRY_PICK_BRANCH --force + + - name: Create Pull Request + run: | + # Prepare and create the PR + pr_title="deps: Merge ${{ env.pr_numbers_in_title }} PRs into $TARGET_BRANCH" + pr_body=$(cat ${{ env.TEMP_DIR }}/pr_body.txt) + + echo "Prepared PR title:" + echo "$pr_title" + echo "Prepared PR body:" + echo "$pr_body" + + # Create the PR using the GitHub API + response=$(curl -s -X POST -H "Authorization: token $BOT_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/${{ github.repository }}/pulls \ + -d "$(jq -n --arg title "$pr_title" \ + --arg head "$CHERRY_PICK_BRANCH" \ + --arg base "$TARGET_BRANCH" \ + --arg body "$pr_body" \ + '{title: $title, head: $head, base: $base, body: $body}')") + + pr_number=$(echo "$response" | jq -r '.number') + echo "$pr_number" > ${{ env.TEMP_DIR }}/created_pr_number.txt + echo "Created PR #$pr_number" + + - name: Add Label to Created Pull Request + run: | + # Add 'milestone-merge' label to the created PR + pr_number=$(cat ${{ env.TEMP_DIR }}/created_pr_number.txt) + echo "Adding label to PR #$pr_number" + + curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -d '{"labels": ["milestone-merge"]}' \ + "https://api.github.com/repos/${{ github.repository }}/issues/$pr_number/labels" + + echo "Added 'milestone-merge' label to PR #$pr_number." diff --git a/README.md b/README.md index a99559cdb0..6b1779b252 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ [![Best Practices](https://img.shields.io/badge/Best%20Practices-purple?style=for-the-badge)](https://www.bestpractices.dev/projects/8045) [![Good First Issues](https://img.shields.io/github/issues/openimsdk/open-im-server/good%20first%20issue?style=for-the-badge&logo=github)](https://github.com/openimsdk/open-im-server/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) [![Language](https://img.shields.io/badge/Language-Go-blue.svg?style=for-the-badge&logo=go&logoColor=white)](https://golang.org/) +[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20OpenIM%20Guru-006BFF?style=for-the-badge)](https://gurubase.io/g/openim)

diff --git a/config/notification.yml b/config/notification.yml index 85ca91af18..ba5ca1c21a 100644 --- a/config/notification.yml +++ b/config/notification.yml @@ -1,20 +1,3 @@ -# Copyright © 2023 OpenIM. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Determines if a message should be sent. If set to false, it triggers a silent sync without a message. If true, it requires triggering a conversation. -# For rpc notification, send twice: once as a message and once as a notification. -# The options field 'isNotification' indicates if it's a notification. groupCreated: isSendMsg: true # Reliability level of the message sending. @@ -309,9 +292,9 @@ userInfoUpdated: unreadCount: false offlinePush: enable: true - title: Remove a blocked user - desc: Remove a blocked user - ext: Remove a blocked user + title: userInfo updated + desc: userInfo updated + ext: userInfo updated userStatusChanged: isSendMsg: false diff --git a/go.mod b/go.mod index 5e5a8b5bea..7cd4e4550f 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.47 + github.com/openimsdk/protocol v0.0.72-alpha.51 github.com/openimsdk/tools v0.0.50-alpha.16 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index 53109c8908..7cc45616f5 100644 --- a/go.sum +++ b/go.sum @@ -319,8 +319,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y= github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.47 h1:FGHnEwsA05GxT3vnz7YH3fbVkuoO3P71ZZgkQQ71MjA= -github.com/openimsdk/protocol v0.0.72-alpha.47/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= +github.com/openimsdk/protocol v0.0.72-alpha.51 h1:G5Yjndp/FRyOJWhoQcSF2x2GvYiAIlqN0vjkvjUPycU= +github.com/openimsdk/protocol v0.0.72-alpha.51/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8= github.com/openimsdk/tools v0.0.50-alpha.16 h1:bC1AQvJMuOHtZm8LZRvN8L5mH1Ws2VYdL+TLTs1iGSc= github.com/openimsdk/tools v0.0.50-alpha.16/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/api/router.go b/internal/api/router.go index 17c9989120..560516d303 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -198,13 +198,6 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En objectGroup.POST("/initiate_form_data", t.InitiateFormData) objectGroup.POST("/complete_form_data", t.CompleteFormData) objectGroup.GET("/*name", t.ObjectRedirect) - - applicationGroup := r.Group("application") - applicationGroup.POST("/add_version", t.AddApplicationVersion) - applicationGroup.POST("/update_version", t.UpdateApplicationVersion) - applicationGroup.POST("/delete_version", t.DeleteApplicationVersion) - applicationGroup.POST("/latest_version", t.LatestApplicationVersion) - applicationGroup.POST("/page_versions", t.PageApplicationVersion) } // Message msgGroup := r.Group("/msg") @@ -297,6 +290,4 @@ func GinParseToken(authRPC *rpcclient.Auth) gin.HandlerFunc { var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", - "/application/latest_version", - "/application/page_versions", } diff --git a/internal/api/third.go b/internal/api/third.go index 56661ba89d..6baa70ee5d 100644 --- a/internal/api/third.go +++ b/internal/api/third.go @@ -170,23 +170,3 @@ func (o *ThirdApi) SearchLogs(c *gin.Context) { func (o *ThirdApi) GetPrometheus(c *gin.Context) { c.Redirect(http.StatusFound, o.GrafanaUrl) } - -func (o *ThirdApi) LatestApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.LatestApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) AddApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.AddApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) UpdateApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.UpdateApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) DeleteApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.DeleteApplicationVersion, o.Client, c) -} - -func (o *ThirdApi) PageApplicationVersion(c *gin.Context) { - a2r.Call(third.ThirdClient.PageApplicationVersion, o.Client, c) -} diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 7dc2ebeea0..f11cfde1af 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -128,6 +128,7 @@ func (m *MsgTransfer) Start(index int, config *Config) error { go m.historyCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyCH) go m.historyMongoCH.historyConsumerGroup.RegisterHandleAndConsumer(m.ctx, m.historyMongoCH) + go m.historyCH.HandleUserHasReadSeqMessages(m.ctx) err := m.historyCH.redisMessageBatches.Start() if err != nil { return err @@ -157,12 +158,14 @@ func (m *MsgTransfer) Start(index int, config *Config) error { // graceful close kafka client. m.cancel() m.historyCH.redisMessageBatches.Close() + m.historyCH.Close() m.historyCH.historyConsumerGroup.Close() m.historyMongoCH.historyConsumerGroup.Close() return nil case <-netDone: m.cancel() m.historyCH.redisMessageBatches.Close() + m.historyCH.Close() m.historyCH.historyConsumerGroup.Close() m.historyMongoCH.historyConsumerGroup.Close() close(netDone) diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index b0078649cb..84453c8df4 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -18,8 +18,10 @@ import ( "context" "encoding/json" "errors" + "github.com/openimsdk/open-im-server/v3/pkg/common/prommetrics" "strconv" "strings" + "sync" "time" "github.com/IBM/sarama" @@ -40,11 +42,12 @@ import ( ) const ( - size = 500 - mainDataBuffer = 500 - subChanBuffer = 50 - worker = 50 - interval = 100 * time.Millisecond + size = 500 + mainDataBuffer = 500 + subChanBuffer = 50 + worker = 50 + interval = 100 * time.Millisecond + hasReadChanBuffer = 1000 ) type ContextMsg struct { @@ -52,14 +55,23 @@ type ContextMsg struct { ctx context.Context } +// This structure is used for asynchronously writing the sender’s read sequence (seq) regarding a message into MongoDB. +// For example, if the sender sends a message with a seq of 10, then their own read seq for this conversation should be set to 10. +type userHasReadSeq struct { + conversationID string + userHasReadMap map[string]int64 +} + type OnlineHistoryRedisConsumerHandler struct { historyConsumerGroup *kafka.MConsumerGroup redisMessageBatches *batcher.Batcher[sarama.ConsumerMessage] - msgTransferDatabase controller.MsgTransferDatabase - conversationRpcClient *rpcclient.ConversationRpcClient - groupRpcClient *rpcclient.GroupRpcClient + msgTransferDatabase controller.MsgTransferDatabase + conversationRpcClient *rpcclient.ConversationRpcClient + groupRpcClient *rpcclient.GroupRpcClient + conversationUserHasReadChan chan *userHasReadSeq + wg sync.WaitGroup } func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database controller.MsgTransferDatabase, @@ -70,6 +82,8 @@ func NewOnlineHistoryRedisConsumerHandler(kafkaConf *config.Kafka, database cont } var och OnlineHistoryRedisConsumerHandler och.msgTransferDatabase = database + och.conversationUserHasReadChan = make(chan *userHasReadSeq, hasReadChanBuffer) + och.wg.Add(1) b := batcher.New[sarama.ConsumerMessage]( batcher.WithSize(size), @@ -115,25 +129,25 @@ func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID } func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, msgs []*ContextMsg) { - type seqKey struct { - conversationID string - userID string - } - var readSeq map[seqKey]int64 + + var conversationID string + var userSeqMap map[string]int64 for _, msg := range msgs { if msg.message.ContentType != constant.HasReadReceipt { continue } var elem sdkws.NotificationElem if err := json.Unmarshal(msg.message.Content, &elem); err != nil { - log.ZError(ctx, "handlerConversationRead Unmarshal NotificationElem msg err", err, "msg", msg) + log.ZWarn(ctx, "handlerConversationRead Unmarshal NotificationElem msg err", err, "msg", msg) continue } var tips sdkws.MarkAsReadTips if err := json.Unmarshal([]byte(elem.Detail), &tips); err != nil { - log.ZError(ctx, "handlerConversationRead Unmarshal MarkAsReadTips msg err", err, "msg", msg) + log.ZWarn(ctx, "handlerConversationRead Unmarshal MarkAsReadTips msg err", err, "msg", msg) continue } + //The conversation ID for each batch of messages processed by the batcher is the same. + conversationID = tips.ConversationID if len(tips.Seqs) > 0 { for _, seq := range tips.Seqs { if tips.HasReadSeq < seq { @@ -146,26 +160,25 @@ func (och *OnlineHistoryRedisConsumerHandler) doSetReadSeq(ctx context.Context, if tips.HasReadSeq < 0 { continue } - if readSeq == nil { - readSeq = make(map[seqKey]int64) - } - key := seqKey{ - conversationID: tips.ConversationID, - userID: tips.MarkAsReadUserID, + if userSeqMap == nil { + userSeqMap = make(map[string]int64) } - if readSeq[key] > tips.HasReadSeq { + + if userSeqMap[tips.MarkAsReadUserID] > tips.HasReadSeq { continue } - readSeq[key] = tips.HasReadSeq + userSeqMap[tips.MarkAsReadUserID] = tips.HasReadSeq } - if readSeq == nil { + if userSeqMap == nil { return } - for key, seq := range readSeq { - if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, key.userID, key.conversationID, seq); err != nil { - log.ZError(ctx, "set read seq to db error", err, "userID", key.userID, "conversationID", key.conversationID, "seq", seq) - } + if len(conversationID) == 0 { + log.ZWarn(ctx, "conversation err", nil, "conversationID", conversationID) } + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, conversationID, userSeqMap); err != nil { + log.ZWarn(ctx, "set read seq to db error", err, "conversationID", conversationID, "userSeqMap", userSeqMap) + } + } func (och *OnlineHistoryRedisConsumerHandler) parseConsumerMessages(ctx context.Context, consumerMessages []*sarama.ConsumerMessage) []*ContextMsg { @@ -250,12 +263,21 @@ func (och *OnlineHistoryRedisConsumerHandler) handleMsg(ctx context.Context, key } if len(storageMessageList) > 0 { msg := storageMessageList[0] - lastSeq, isNewConversation, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, isNewConversation, userSeqMap, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil && !errors.Is(errs.Unwrap(err), redis.Nil) { - log.ZError(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) + log.ZWarn(ctx, "batch data insert to redis err", err, "storageMsgList", storageMessageList) return } log.ZInfo(ctx, "BatchInsertChat2Cache end") + err = och.msgTransferDatabase.SetHasReadSeqs(ctx, conversationID, userSeqMap) + if err != nil { + log.ZWarn(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) + prommetrics.SeqSetFailedCounter.Inc() + } + och.conversationUserHasReadChan <- &userHasReadSeq{ + conversationID: conversationID, + userHasReadMap: userSeqMap, + } if isNewConversation { switch msg.SessionType { @@ -308,7 +330,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con storageMessageList = append(storageMessageList, msg.message) } if len(storageMessageList) > 0 { - lastSeq, _, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) + lastSeq, _, _, err := och.msgTransferDatabase.BatchInsertChat2Cache(ctx, conversationID, storageMessageList) if err != nil { log.ZError(ctx, "notification batch insert to redis error", err, "conversationID", conversationID, "storageList", storageMessageList) @@ -323,6 +345,21 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con och.toPushTopic(ctx, key, conversationID, storageList) } } +func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { + defer och.wg.Done() + + for msg := range och.conversationUserHasReadChan { + if err := och.msgTransferDatabase.SetHasReadSeqToDB(ctx, msg.conversationID, msg.userHasReadMap); err != nil { + log.ZWarn(ctx, "set read seq to db error", err, "conversationID", msg.conversationID, "userSeqMap", msg.userHasReadMap) + } + } + + log.ZInfo(ctx, "Channel closed, exiting handleUserHasReadSeqMessages") +} +func (och *OnlineHistoryRedisConsumerHandler) Close() { + close(och.conversationUserHasReadChan) + och.wg.Wait() +} func (och *OnlineHistoryRedisConsumerHandler) toPushTopic(ctx context.Context, key, conversationID string, msgs []*ContextMsg) { for _, v := range msgs { diff --git a/internal/push/callback.go b/internal/push/callback.go index 8897295827..f8e17bb8c0 100644 --- a/internal/push/callback.go +++ b/internal/push/callback.go @@ -24,7 +24,6 @@ import ( "github.com/openimsdk/protocol/constant" "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/mcontext" - "github.com/openimsdk/tools/utils/datautil" ) func (c *ConsumerHandler) webhookBeforeOfflinePush(ctx context.Context, before *config.BeforeConfig, userIDs []string, msg *sdkws.MsgData, offlinePushUserIDs *[]string) error { @@ -70,7 +69,7 @@ func (c *ConsumerHandler) webhookBeforeOfflinePush(ctx context.Context, before * func (c *ConsumerHandler) webhookBeforeOnlinePush(ctx context.Context, before *config.BeforeConfig, userIDs []string, msg *sdkws.MsgData) error { return webhook.WithCondition(ctx, before, func(ctx context.Context) error { - if datautil.Contain(msg.SendID, userIDs...) || msg.ContentType == constant.Typing { + if msg.ContentType == constant.Typing { return nil } req := callbackstruct.CallbackBeforePushReq{ diff --git a/internal/rpc/group/group.go b/internal/rpc/group/group.go index ef917d5395..b5ab1b2097 100644 --- a/internal/rpc/group/group.go +++ b/internal/rpc/group/group.go @@ -1026,7 +1026,7 @@ func (g *groupServer) SetGroupInfo(ctx context.Context, req *pbgroup.SetGroupInf } num := len(update) if req.GroupInfoForSet.Notification != "" { - num-- + num -= 3 func() { conversation := &pbconversation.ConversationReq{ ConversationID: msgprocessor.GetConversationIDBySessionType(constant.ReadGroupChatType, req.GroupInfoForSet.GroupID), @@ -1133,8 +1133,9 @@ func (g *groupServer) SetGroupInfoEx(ctx context.Context, req *pbgroup.SetGroupI } num := len(updatedData) + if req.Notification != nil { - num-- + num -= 3 if req.Notification.Value != "" { func() { @@ -1219,7 +1220,7 @@ func (g *groupServer) TransferGroupOwner(ctx context.Context, req *pbgroup.Trans } } - if newOwner.MuteEndTime != time.Unix(0, 0) { + if newOwner.MuteEndTime.After(time.Now()) { if _, err := g.CancelMuteGroupMember(ctx, &pbgroup.CancelMuteGroupMemberReq{ GroupID: group.GroupID, UserID: req.NewOwnerUserID}); err != nil { @@ -1810,7 +1811,6 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req } if req.UserID != opUserID { - req.UserID = mcontext.GetOpUserID(ctx) adminIDs, err := g.db.GetGroupRoleLevelMemberIDs(ctx, req.GroupID, constant.GroupAdmin) if err != nil { return nil, err @@ -1819,10 +1819,11 @@ func (g *groupServer) GetSpecifiedUserGroupRequestInfo(ctx context.Context, req adminIDs = append(adminIDs, owners[0].UserID) adminIDs = append(adminIDs, g.config.Share.IMAdminUserID...) - if !datautil.Contain(req.UserID, adminIDs...) { + if !datautil.Contain(opUserID, adminIDs...) { return nil, errs.ErrNoPermission.WrapMsg("opUser no permission") } } + requests, err := g.db.FindGroupRequests(ctx, req.GroupID, []string{req.UserID}) if err != nil { return nil, err diff --git a/internal/rpc/third/application.go b/internal/rpc/third/application.go deleted file mode 100644 index a6556055c1..0000000000 --- a/internal/rpc/third/application.go +++ /dev/null @@ -1,117 +0,0 @@ -package third - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/authverify" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/third" - "github.com/openimsdk/tools/errs" - "github.com/openimsdk/tools/utils/datautil" - "github.com/redis/go-redis/v9" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "time" -) - -func IsNotFound(err error) bool { - switch errs.Unwrap(err) { - case redis.Nil, mongo.ErrNoDocuments: - return true - default: - return false - } -} - -func (t *thirdServer) db2pbApplication(val *model.Application) *third.ApplicationVersion { - return &third.ApplicationVersion{ - Id: val.ID.Hex(), - Platform: val.Platform, - Version: val.Version, - Url: val.Url, - Text: val.Text, - Force: val.Force, - Latest: val.Latest, - CreateTime: val.CreateTime.UnixMilli(), - } -} - -func (t *thirdServer) LatestApplicationVersion(ctx context.Context, req *third.LatestApplicationVersionReq) (*third.LatestApplicationVersionResp, error) { - res, err := t.applicationDatabase.LatestVersion(ctx, req.Platform) - if err == nil { - return &third.LatestApplicationVersionResp{Version: t.db2pbApplication(res)}, nil - } else if IsNotFound(err) { - return &third.LatestApplicationVersionResp{}, nil - } else { - return nil, err - } -} - -func (t *thirdServer) AddApplicationVersion(ctx context.Context, req *third.AddApplicationVersionReq) (*third.AddApplicationVersionResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { - return nil, err - } - val := &model.Application{ - ID: primitive.NewObjectID(), - Platform: req.Platform, - Version: req.Version, - Url: req.Url, - Text: req.Text, - Force: req.Force, - Latest: req.Latest, - CreateTime: time.Now(), - } - if err := t.applicationDatabase.AddVersion(ctx, val); err != nil { - return nil, err - } - return &third.AddApplicationVersionResp{}, nil -} - -func (t *thirdServer) UpdateApplicationVersion(ctx context.Context, req *third.UpdateApplicationVersionReq) (*third.UpdateApplicationVersionResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { - return nil, err - } - oid, err := primitive.ObjectIDFromHex(req.Id) - if err != nil { - return nil, errs.ErrArgs.WrapMsg("invalid id " + err.Error()) - } - update := make(map[string]any) - putUpdate(update, "platform", req.Platform) - putUpdate(update, "version", req.Version) - putUpdate(update, "url", req.Url) - putUpdate(update, "text", req.Text) - putUpdate(update, "force", req.Force) - putUpdate(update, "latest", req.Latest) - if err := t.applicationDatabase.UpdateVersion(ctx, oid, update); err != nil { - return nil, err - } - return &third.UpdateApplicationVersionResp{}, nil -} - -func (t *thirdServer) DeleteApplicationVersion(ctx context.Context, req *third.DeleteApplicationVersionReq) (*third.DeleteApplicationVersionResp, error) { - if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { - return nil, err - } - ids := make([]primitive.ObjectID, 0, len(req.Id)) - for _, id := range req.Id { - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - return nil, errs.ErrArgs.WrapMsg("invalid id " + err.Error()) - } - ids = append(ids, oid) - } - if err := t.applicationDatabase.DeleteVersion(ctx, ids); err != nil { - return nil, err - } - return &third.DeleteApplicationVersionResp{}, nil -} - -func (t *thirdServer) PageApplicationVersion(ctx context.Context, req *third.PageApplicationVersionReq) (*third.PageApplicationVersionResp, error) { - total, res, err := t.applicationDatabase.PageVersion(ctx, req.Platform, req.Pagination) - if err != nil { - return nil, err - } - return &third.PageApplicationVersionResp{ - Total: total, - Versions: datautil.Slice(res, t.db2pbApplication), - }, nil -} diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index c6b588d8d4..d37689b313 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -38,13 +38,12 @@ import ( ) type thirdServer struct { - thirdDatabase controller.ThirdDatabase - s3dataBase controller.S3Database - userRpcClient rpcclient.UserRpcClient - defaultExpire time.Duration - config *Config - minio *minio.Minio - applicationDatabase controller.ApplicationDatabase + thirdDatabase controller.ThirdDatabase + s3dataBase controller.S3Database + userRpcClient rpcclient.UserRpcClient + defaultExpire time.Duration + config *Config + minio *minio.Minio } type Config struct { @@ -75,10 +74,6 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg if err != nil { return err } - applicationMgo, err := mgo.NewApplicationMgo(mgocli.GetDB()) - if err != nil { - return err - } // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable @@ -104,13 +99,12 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg } localcache.InitLocalCache(&config.LocalCacheConfig) third.RegisterThirdServer(server, &thirdServer{ - thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), - userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), - s3dataBase: controller.NewS3Database(rdb, o, s3db), - defaultExpire: time.Hour * 24 * 7, - config: config, - minio: minioCli, - applicationDatabase: controller.NewApplicationDatabase(applicationMgo, redis.NewApplicationRedisCache(applicationMgo, rdb)), + thirdDatabase: controller.NewThirdDatabase(redis.NewThirdCache(rdb), logdb), + userRpcClient: rpcclient.NewUserRpcClient(client, config.Share.RpcRegisterName.User, config.Share.IMAdminUserID), + s3dataBase: controller.NewS3Database(rdb, o, s3db), + defaultExpire: time.Hour * 24 * 7, + config: config, + minio: minioCli, }) return nil } diff --git a/pkg/common/storage/cache/application.go b/pkg/common/storage/cache/application.go deleted file mode 100644 index 588732ec8d..0000000000 --- a/pkg/common/storage/cache/application.go +++ /dev/null @@ -1,11 +0,0 @@ -package cache - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" -) - -type ApplicationCache interface { - LatestVersion(ctx context.Context, platform string) (*model.Application, error) - DeleteCache(ctx context.Context, platforms []string) error -} diff --git a/pkg/common/storage/cache/cachekey/application.go b/pkg/common/storage/cache/cachekey/application.go deleted file mode 100644 index 032adba3c1..0000000000 --- a/pkg/common/storage/cache/cachekey/application.go +++ /dev/null @@ -1,9 +0,0 @@ -package cachekey - -const ( - ApplicationLatestVersion = "APPLICATION_LATEST_VERSION:" -) - -func GetApplicationLatestVersionKey(platform string) string { - return ApplicationLatestVersion + platform -} diff --git a/pkg/common/storage/cache/redis/application.go b/pkg/common/storage/cache/redis/application.go deleted file mode 100644 index 4a7a4ced67..0000000000 --- a/pkg/common/storage/cache/redis/application.go +++ /dev/null @@ -1,43 +0,0 @@ -package redis - -import ( - "context" - "github.com/dtm-labs/rockscache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/utils/datautil" - "github.com/redis/go-redis/v9" - "time" -) - -func NewApplicationRedisCache(db database.Application, rdb redis.UniversalClient) *ApplicationRedisCache { - return &ApplicationRedisCache{ - db: db, - rcClient: rockscache.NewClient(rdb, *GetRocksCacheOptions()), - deleter: NewBatchDeleterRedis(rdb, GetRocksCacheOptions(), nil), - expireTime: time.Hour * 24 * 7, - } -} - -type ApplicationRedisCache struct { - db database.Application - rcClient *rockscache.Client - deleter *BatchDeleterRedis - expireTime time.Duration -} - -func (a *ApplicationRedisCache) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { - return getCache(ctx, a.rcClient, cachekey.GetApplicationLatestVersionKey(platform), a.expireTime, func(ctx context.Context) (*model.Application, error) { - return a.db.LatestVersion(ctx, platform) - }) -} - -func (a *ApplicationRedisCache) DeleteCache(ctx context.Context, platforms []string) error { - if len(platforms) == 0 { - return nil - } - return a.deleter.ExecDelWithKeys(ctx, datautil.Slice(platforms, func(platform string) string { - return cachekey.GetApplicationLatestVersionKey(platform) - })) -} diff --git a/pkg/common/storage/controller/application.go b/pkg/common/storage/controller/application.go deleted file mode 100644 index 72bca07efa..0000000000 --- a/pkg/common/storage/controller/application.go +++ /dev/null @@ -1,69 +0,0 @@ -package controller - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type ApplicationDatabase interface { - LatestVersion(ctx context.Context, platform string) (*model.Application, error) - AddVersion(ctx context.Context, val *model.Application) error - UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error - DeleteVersion(ctx context.Context, id []primitive.ObjectID) error - PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) -} - -func NewApplicationDatabase(db database.Application, cache cache.ApplicationCache) ApplicationDatabase { - return &applicationDatabase{db: db, cache: cache} -} - -type applicationDatabase struct { - db database.Application - cache cache.ApplicationCache -} - -func (a *applicationDatabase) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { - return a.cache.LatestVersion(ctx, platform) -} - -func (a *applicationDatabase) AddVersion(ctx context.Context, val *model.Application) error { - if err := a.db.AddVersion(ctx, val); err != nil { - return err - } - return a.cache.DeleteCache(ctx, []string{val.Platform}) -} - -func (a *applicationDatabase) UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error { - platforms, err := a.db.FindPlatform(ctx, []primitive.ObjectID{id}) - if err != nil { - return err - } - if err := a.db.UpdateVersion(ctx, id, update); err != nil { - return err - } - if p, ok := update["platform"]; ok { - if val, ok := p.(string); ok { - platforms = append(platforms, val) - } - } - return a.cache.DeleteCache(ctx, platforms) -} - -func (a *applicationDatabase) DeleteVersion(ctx context.Context, id []primitive.ObjectID) error { - platforms, err := a.db.FindPlatform(ctx, id) - if err != nil { - return err - } - if err := a.db.DeleteVersion(ctx, id); err != nil { - return err - } - return a.cache.DeleteCache(ctx, platforms) -} - -func (a *applicationDatabase) PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) { - return a.db.PageVersion(ctx, platforms, page) -} diff --git a/pkg/common/storage/controller/msg_transfer.go b/pkg/common/storage/controller/msg_transfer.go index c6013dbc12..1ecd786aa3 100644 --- a/pkg/common/storage/controller/msg_transfer.go +++ b/pkg/common/storage/controller/msg_transfer.go @@ -24,8 +24,11 @@ type MsgTransferDatabase interface { DeleteMessagesFromCache(ctx context.Context, conversationID string, seqs []int64) error // BatchInsertChat2Cache increments the sequence number and then batch inserts messages into the cache. - BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, err error) - SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error + BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNewConversation bool, userHasReadMap map[string]int64, err error) + + SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error + + SetHasReadSeqToDB(ctx context.Context, conversationID string, userSeqMap map[string]int64) error // to mq MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) @@ -219,18 +222,18 @@ func (db *msgTransferDatabase) DeleteMessagesFromCache(ctx context.Context, conv return db.msg.DeleteMessagesFromCache(ctx, conversationID, seqs) } -func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, err error) { +func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conversationID string, msgs []*sdkws.MsgData) (seq int64, isNew bool, userHasReadMap map[string]int64, err error) { lenList := len(msgs) if int64(lenList) > db.msgTable.GetSingleGocMsgNum() { - return 0, false, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() + return 0, false, nil, errs.New("message count exceeds limit", "limit", db.msgTable.GetSingleGocMsgNum()).Wrap() } if lenList < 1 { - return 0, false, errs.New("no messages to insert", "minCount", 1).Wrap() + return 0, false, nil, errs.New("no messages to insert", "minCount", 1).Wrap() } currentMaxSeq, err := db.seqConversation.Malloc(ctx, conversationID, int64(len(msgs))) if err != nil { log.ZError(ctx, "storage.seq.Malloc", err) - return 0, false, err + return 0, false, nil, err } isNew = currentMaxSeq == 0 lastMaxSeq := currentMaxSeq @@ -248,25 +251,25 @@ func (db *msgTransferDatabase) BatchInsertChat2Cache(ctx context.Context, conver } else { prommetrics.MsgInsertRedisSuccessCounter.Inc() } - err = db.setHasReadSeqs(ctx, conversationID, userSeqMap) - if err != nil { - log.ZError(ctx, "SetHasReadSeqs error", err, "userSeqMap", userSeqMap, "conversationID", conversationID) - prommetrics.SeqSetFailedCounter.Inc() - } - return lastMaxSeq, isNew, errs.Wrap(err) + return lastMaxSeq, isNew, userSeqMap, errs.Wrap(err) } -func (db *msgTransferDatabase) setHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { +func (db *msgTransferDatabase) SetHasReadSeqs(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { for userID, seq := range userSeqMap { - if err := db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { + if err := db.seqUser.SetUserReadSeq(ctx, conversationID, userID, seq); err != nil { return err } } return nil } -func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error { - return db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, hasReadSeq) +func (db *msgTransferDatabase) SetHasReadSeqToDB(ctx context.Context, conversationID string, userSeqMap map[string]int64) error { + for userID, seq := range userSeqMap { + if err := db.seqUser.SetUserReadSeqToDB(ctx, conversationID, userID, seq); err != nil { + return err + } + } + return nil } func (db *msgTransferDatabase) MsgToPushMQ(ctx context.Context, key, conversationID string, msg2mq *sdkws.MsgData) (int32, int64, error) { diff --git a/pkg/common/storage/database/application.go b/pkg/common/storage/database/application.go deleted file mode 100644 index c98ae74c82..0000000000 --- a/pkg/common/storage/database/application.go +++ /dev/null @@ -1,17 +0,0 @@ -package database - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -type Application interface { - LatestVersion(ctx context.Context, platform string) (*model.Application, error) - AddVersion(ctx context.Context, val *model.Application) error - UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error - DeleteVersion(ctx context.Context, id []primitive.ObjectID) error - PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) - FindPlatform(ctx context.Context, id []primitive.ObjectID) ([]string, error) -} diff --git a/pkg/common/storage/database/mgo/application.go b/pkg/common/storage/database/mgo/application.go deleted file mode 100644 index e59c0560ab..0000000000 --- a/pkg/common/storage/database/mgo/application.go +++ /dev/null @@ -1,82 +0,0 @@ -package mgo - -import ( - "context" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func NewApplicationMgo(db *mongo.Database) (*ApplicationMgo, error) { - coll := db.Collection("application") - _, err := coll.Indexes().CreateMany(context.Background(), []mongo.IndexModel{ - { - Keys: bson.D{ - {Key: "platform", Value: 1}, - {Key: "version", Value: 1}, - }, - Options: options.Index().SetUnique(true), - }, - { - Keys: bson.D{ - {Key: "latest", Value: -1}, - }, - }, - }) - if err != nil { - return nil, err - } - return &ApplicationMgo{coll: coll}, nil -} - -type ApplicationMgo struct { - coll *mongo.Collection -} - -func (a *ApplicationMgo) sort() any { - return bson.D{{"latest", -1}, {"_id", -1}} -} - -func (a *ApplicationMgo) LatestVersion(ctx context.Context, platform string) (*model.Application, error) { - return mongoutil.FindOne[*model.Application](ctx, a.coll, bson.M{"platform": platform}, options.FindOne().SetSort(a.sort())) -} - -func (a *ApplicationMgo) AddVersion(ctx context.Context, val *model.Application) error { - if val.ID.IsZero() { - val.ID = primitive.NewObjectID() - } - return mongoutil.InsertMany(ctx, a.coll, []*model.Application{val}) -} - -func (a *ApplicationMgo) UpdateVersion(ctx context.Context, id primitive.ObjectID, update map[string]any) error { - if len(update) == 0 { - return nil - } - return mongoutil.UpdateOne(ctx, a.coll, bson.M{"_id": id}, bson.M{"$set": update}, true) -} - -func (a *ApplicationMgo) DeleteVersion(ctx context.Context, id []primitive.ObjectID) error { - if len(id) == 0 { - return nil - } - return mongoutil.DeleteMany(ctx, a.coll, bson.M{"_id": bson.M{"$in": id}}) -} - -func (a *ApplicationMgo) PageVersion(ctx context.Context, platforms []string, page pagination.Pagination) (int64, []*model.Application, error) { - filter := bson.M{} - if len(platforms) > 0 { - filter["platform"] = bson.M{"$in": platforms} - } - return mongoutil.FindPage[*model.Application](ctx, a.coll, filter, page, options.Find().SetSort(a.sort())) -} - -func (a *ApplicationMgo) FindPlatform(ctx context.Context, id []primitive.ObjectID) ([]string, error) { - if len(id) == 0 { - return nil, nil - } - return mongoutil.Find[string](ctx, a.coll, bson.M{"_id": bson.M{"$in": id}}, options.Find().SetProjection(bson.M{"_id": 0, "platform": 1})) -} diff --git a/pkg/common/storage/model/application.go b/pkg/common/storage/model/application.go index f5bae2be65..b09b0e8948 100644 --- a/pkg/common/storage/model/application.go +++ b/pkg/common/storage/model/application.go @@ -8,6 +8,7 @@ import ( type Application struct { ID primitive.ObjectID `bson:"_id"` Platform string `bson:"platform"` + Hot bool `bson:"hot"` Version string `bson:"version"` Url string `bson:"url"` Text string `bson:"text"` From b49a8dd99d14d24f1155d89a892ffbcdd6b472ef Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 4 Nov 2024 11:36:20 +0800 Subject: [PATCH 10/12] feat: support text ping pong --- internal/msggateway/client.go | 30 +++++++++++++++++++++++--- internal/msggateway/message_handler.go | 11 ++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index bc06fa9507..99598d5213 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -16,6 +16,7 @@ package msggateway import ( "context" + "encoding/json" "fmt" "runtime/debug" "sync" @@ -159,9 +160,12 @@ func (c *Client) readMessage() { return } case MessageText: - c.closedErr = ErrNotSupportMessageProtocol - return - + _ = c.conn.SetReadDeadline(pongWait) + parseDataErr := c.handlerTextMessage(message) + if parseDataErr != nil { + c.closedErr = parseDataErr + return + } case PingMessage: err := c.writePongMsg("") log.ZError(c.ctx, "writePongMsg", err) @@ -419,3 +423,23 @@ func (c *Client) writePongMsg(appData string) error { return errs.Wrap(err) } + +func (c *Client) handlerTextMessage(b []byte) error { + var msg TextMessage + if err := json.Unmarshal(b, &msg); err != nil { + return err + } + switch msg.Type { + case TextPong: + return nil + case TextPing: + msg.Type = TextPong + msgData, err := json.Marshal(msg) + if err != nil { + return err + } + return c.conn.WriteMessage(MessageText, msgData) + default: + return fmt.Errorf("not support message type %s", msg.Type) + } +} diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 4b78c10048..5407ba90cb 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -16,6 +16,7 @@ package msggateway import ( "context" + "encoding/json" "sync" "github.com/go-playground/validator/v10" @@ -31,6 +32,16 @@ import ( "github.com/openimsdk/tools/utils/jsonutil" ) +const ( + TextPing = "ping" + TextPong = "pong" +) + +type TextMessage struct { + Type string `json:"type"` + Body json.RawMessage `json:"body"` +} + type Req struct { ReqIdentifier int32 `json:"reqIdentifier" validate:"required"` Token string `json:"token"` From f185c981f764881d15354a0097793b251b44577a Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 5 Nov 2024 16:24:07 +0800 Subject: [PATCH 11/12] feat: support text ping pong --- internal/msggateway/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 99598d5213..48cbf2b6ad 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -438,6 +438,9 @@ func (c *Client) handlerTextMessage(b []byte) error { if err != nil { return err } + if err := c.conn.SetWriteDeadline(writeWait); err != nil { + return err + } return c.conn.WriteMessage(MessageText, msgData) default: return fmt.Errorf("not support message type %s", msg.Type) From 97d10f99e4916c639ed01fc2153040eca799897e Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 14 Nov 2024 11:32:35 +0800 Subject: [PATCH 12/12] feat: gob json encoder --- internal/msggateway/client.go | 17 +++++++++++--- internal/msggateway/constant.go | 6 +++++ internal/msggateway/context.go | 6 ++++- internal/msggateway/encoder.go | 38 ++++++++++++++++++++++++++----- internal/msggateway/hub_server.go | 15 +++--------- internal/msggateway/ws_server.go | 16 ++++--------- 6 files changed, 64 insertions(+), 34 deletions(-) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 48cbf2b6ad..19548a71c4 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -70,6 +70,8 @@ type Client struct { IsCompress bool `json:"isCompress"` UserID string `json:"userID"` IsBackground bool `json:"isBackground"` + SDKType string `json:"sdkType"` + Encoder Encoder ctx *UserConnContext longConnServer LongConnServer closed atomic.Bool @@ -82,7 +84,7 @@ type Client struct { } // ResetClient updates the client's state with new connection and context information. -func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer LongConnServer) { +func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer LongConnServer, sdkType string) { c.w = new(sync.Mutex) c.conn = conn c.PlatformID = stringutil.StringToInt(ctx.GetPlatformID()) @@ -95,11 +97,20 @@ func (c *Client) ResetClient(ctx *UserConnContext, conn LongConn, longConnServer c.closed.Store(false) c.closedErr = nil c.token = ctx.GetToken() + c.SDKType = sdkType c.hbCtx, c.hbCancel = context.WithCancel(c.ctx) c.subLock = new(sync.Mutex) if c.subUserIDs != nil { clear(c.subUserIDs) } + if c.SDKType == "" { + c.SDKType = GoSDK + } + if c.SDKType == GoSDK { + c.Encoder = NewGobEncoder() + } else { + c.Encoder = NewJsonEncoder() + } c.subUserIDs = make(map[string]struct{}) } @@ -192,7 +203,7 @@ func (c *Client) handleMessage(message []byte) error { var binaryReq = getReq() defer freeReq(binaryReq) - err := c.longConnServer.Decode(message, binaryReq) + err := c.Encoder.Decode(message, binaryReq) if err != nil { return err } @@ -339,7 +350,7 @@ func (c *Client) writeBinaryMsg(resp Resp) error { return nil } - encodedBuf, err := c.longConnServer.Encode(resp) + encodedBuf, err := c.Encoder.Encode(resp) if err != nil { return err } diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index 584cebe1e1..a825c05196 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -27,6 +27,12 @@ const ( GzipCompressionProtocol = "gzip" BackgroundStatus = "isBackground" SendResponse = "isMsgResp" + SDKType = "sdkType" +) + +const ( + GoSDK = "go" + JsSDK = "js" ) const ( diff --git a/internal/msggateway/context.go b/internal/msggateway/context.go index 3909766b1b..f3f168f616 100644 --- a/internal/msggateway/context.go +++ b/internal/msggateway/context.go @@ -193,7 +193,11 @@ func (c *UserConnContext) ParseEssentialArgs() error { _, err := strconv.Atoi(platformIDStr) if err != nil { return servererrs.ErrConnArgsErr.WrapMsg("platformID is not int") - + } + switch sdkType, _ := c.Query(SDKType); sdkType { + case "", GoSDK, JsSDK: + default: + return servererrs.ErrConnArgsErr.WrapMsg("sdkType is not go or js") } return nil } diff --git a/internal/msggateway/encoder.go b/internal/msggateway/encoder.go index 056f462367..6a5936d6d2 100644 --- a/internal/msggateway/encoder.go +++ b/internal/msggateway/encoder.go @@ -15,6 +15,8 @@ package msggateway import ( + "bytes" + "encoding/gob" "encoding/json" "github.com/openimsdk/tools/errs" @@ -27,22 +29,46 @@ type Encoder interface { type GobEncoder struct{} -func NewGobEncoder() *GobEncoder { - return &GobEncoder{} +func NewGobEncoder() Encoder { + return GobEncoder{} } -func (g *GobEncoder) Encode(data any) ([]byte, error) { +func (g GobEncoder) Encode(data any) ([]byte, error) { + var buff bytes.Buffer + enc := gob.NewEncoder(&buff) + if err := enc.Encode(data); err != nil { + return nil, errs.WrapMsg(err, "GobEncoder.Encode failed", "action", "encode") + } + return buff.Bytes(), nil +} + +func (g GobEncoder) Decode(encodeData []byte, decodeData any) error { + buff := bytes.NewBuffer(encodeData) + dec := gob.NewDecoder(buff) + if err := dec.Decode(decodeData); err != nil { + return errs.WrapMsg(err, "GobEncoder.Decode failed", "action", "decode") + } + return nil +} + +type JsonEncoder struct{} + +func NewJsonEncoder() Encoder { + return JsonEncoder{} +} + +func (g JsonEncoder) Encode(data any) ([]byte, error) { b, err := json.Marshal(data) if err != nil { - return nil, errs.New("Encoder.Encode failed", "action", "encode") + return nil, errs.New("JsonEncoder.Encode failed", "action", "encode") } return b, nil } -func (g *GobEncoder) Decode(encodeData []byte, decodeData any) error { +func (g JsonEncoder) Decode(encodeData []byte, decodeData any) error { err := json.Unmarshal(encodeData, decodeData) if err != nil { - return errs.New("Encoder.Decode failed", "action", "decode") + return errs.New("JsonEncoder.Decode failed", "action", "decode") } return nil } diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index cc29876447..23d9150133 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -83,17 +83,11 @@ func NewServer(rpcPort int, longConnServer LongConnServer, conf *Config, ready f return s } -func (s *Server) OnlinePushMsg( - context context.Context, - req *msggateway.OnlinePushMsgReq, -) (*msggateway.OnlinePushMsgResp, error) { +func (s *Server) OnlinePushMsg(context context.Context, req *msggateway.OnlinePushMsgReq) (*msggateway.OnlinePushMsgResp, error) { panic("implement me") } -func (s *Server) GetUsersOnlineStatus( - ctx context.Context, - req *msggateway.GetUsersOnlineStatusReq, -) (*msggateway.GetUsersOnlineStatusResp, error) { +func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUsersOnlineStatusReq) (*msggateway.GetUsersOnlineStatusResp, error) { if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("only app manager") } @@ -221,10 +215,7 @@ func (s *Server) SuperGroupOnlineBatchPushOneMsg(ctx context.Context, req *msgga } } -func (s *Server) KickUserOffline( - ctx context.Context, - req *msggateway.KickUserOfflineReq, -) (*msggateway.KickUserOfflineResp, error) { +func (s *Server) KickUserOffline(ctx context.Context, req *msggateway.KickUserOfflineReq) (*msggateway.KickUserOfflineResp, error) { for _, v := range req.KickUserIDList { clients, _, ok := s.LongConnServer.GetUserPlatformCons(v, int(req.PlatformID)) if !ok { diff --git a/internal/msggateway/ws_server.go b/internal/msggateway/ws_server.go index b92d7eb442..32cdeaee64 100644 --- a/internal/msggateway/ws_server.go +++ b/internal/msggateway/ws_server.go @@ -37,7 +37,6 @@ type LongConnServer interface { SetKickHandlerInfo(i *kickHandler) SubUserOnlineStatus(ctx context.Context, client *Client, data *Req) ([]byte, error) Compressor - Encoder MessageHandler } @@ -61,7 +60,7 @@ type WsServer struct { authClient *rpcclient.Auth disCov discovery.SvcDiscoveryRegistry Compressor - Encoder + //Encoder MessageHandler webhookClient *webhook.Client } @@ -135,7 +134,6 @@ func NewWsServer(msgGatewayConfig *Config, opts ...Option) *WsServer { clients: newUserMap(), subscription: newSubscription(), Compressor: NewGzipCompressor(), - Encoder: NewGobEncoder(), webhookClient: webhook.NewWebhookClient(msgGatewayConfig.WebhooksConfig.URL), } } @@ -278,14 +276,7 @@ func (ws *WsServer) registerClient(client *Client) { wg.Wait() - log.ZDebug( - client.ctx, - "user online", - "online user Num", - ws.onlineUserNum.Load(), - "online user conn Num", - ws.onlineUserConnNum.Load(), - ) + log.ZDebug(client.ctx, "user online", "online user Num", ws.onlineUserNum.Load(), "online user conn Num", ws.onlineUserConnNum.Load()) } func getRemoteAdders(client []*Client) string { @@ -484,7 +475,8 @@ func (ws *WsServer) wsHandler(w http.ResponseWriter, r *http.Request) { // Retrieve a client object from the client pool, reset its state, and associate it with the current WebSocket long connection client := ws.clientPool.Get().(*Client) - client.ResetClient(connContext, wsLongConn, ws) + sdkType, _ := connContext.Query(SDKType) + client.ResetClient(connContext, wsLongConn, ws, sdkType) // Register the client with the server and start message processing ws.registerChan <- client