Merge branch 'openimsdk:main' into main

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/openimsdk/protocol v0.0.69-alpha.50
+	github.com/openimsdk/protocol v0.0.69
 	github.com/openimsdk/tools v0.0.49-alpha.55
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.18.0
@@ -74,6 +74,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/chai2010/webp v1.1.1 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
 	github.com/clbanning/mxj v1.8.4 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect

go.sum

@@ -71,6 +71,8 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk=
+github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
@@ -319,8 +321,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
 github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
 github.com/openimsdk/gomake v0.0.14-alpha.5 h1:VY9c5x515lTfmdhhPjMvR3BBRrRquAUCFsz7t7vbv7Y=
 github.com/openimsdk/gomake v0.0.14-alpha.5/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI=
-github.com/openimsdk/protocol v0.0.69-alpha.50 h1:4r6vY9LsjFrR8AAwORFhijOGmq2vzDH3XTX4wBiw+2M=
-github.com/openimsdk/protocol v0.0.69-alpha.50/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
+github.com/openimsdk/protocol v0.0.69 h1:dVi8meSg8kmUzSH1XQab4MjihqKkkcCAmt1BYXPJuXo=
+github.com/openimsdk/protocol v0.0.69/go.mod h1:OZQA9FR55lseYoN2Ql1XAHYKHJGu7OMNkUbuekrKCM8=
 github.com/openimsdk/tools v0.0.49-alpha.55 h1:KPgC53oqiwZYssLKljhtXbWXifMlTj2SSQEusj4Uf4k=
 github.com/openimsdk/tools v0.0.49-alpha.55/go.mod h1:h1cYmfyaVtgFbKmb1Cfsl8XwUOMTt8ubVUQrdGtsUh4=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=

internal/msggateway/user_map.go

@@ -1,10 +1,6 @@
 package msggateway
 
 import (
-	"context"
-	"fmt"
-	"github.com/openimsdk/protocol/constant"
-	"github.com/openimsdk/tools/log"
 	"github.com/openimsdk/tools/utils/datautil"
 	"sync"
 	"time"
@@ -121,7 +117,6 @@ func (u *userMap) Get(userID string, platformID int) ([]*Client, bool, bool) {
 }
 
 func (u *userMap) Set(userID string, client *Client) {
-	log.ZDebug(context.Background(), "userMap Set", "userID", userID, "platformID", client.PlatformID, "platform", constant.PlatformIDToName(client.PlatformID), "pointer", fmt.Sprintf("%p", client))
 	u.lock.Lock()
 	defer u.lock.Unlock()
 	result, ok := u.data[userID]

pkg/common/storage/database/mgo/msg.go

@@ -377,8 +377,19 @@ func (m *MsgMgo) searchMessageIndex(ctx context.Context, filter any, nextID prim
 	if !nextID.IsZero() {
 		pipeline = append(pipeline, bson.M{"$match": bson.M{"_id": bson.M{"$gt": nextID}}})
 	}
+	coarseFilter := bson.M{
+		"$or": bson.A{
+			bson.M{
+				"doc_id": primitive.Regex{Pattern: "^sg_"},
+			},
+			bson.M{
+				"doc_id": primitive.Regex{Pattern: "^si_"},
+			},
+		},
+	}
 	pipeline = append(pipeline,
 		bson.M{"$sort": bson.M{"_id": 1}},
+		bson.M{"$match": coarseFilter},
 		bson.M{"$match": filter},
 		bson.M{"$limit": limit},
 		bson.M{

pkg/rpccache/common.go

@@ -14,6 +14,11 @@
 
 package rpccache
 
+import (
+	"github.com/openimsdk/tools/errs"
+	"google.golang.org/protobuf/proto"
+)
+
 func newListMap[V comparable](values []V, err error) (*listMap[V], error) {
 	if err != nil {
 		return nil, err
@@ -32,3 +37,41 @@ type listMap[V comparable] struct {
 	List []V
 	Map  map[V]struct{}
 }
+
+func respProtoMarshal(resp proto.Message, err error) ([]byte, error) {
+	if err != nil {
+		return nil, err
+	}
+	return proto.Marshal(resp)
+}
+
+func cacheUnmarshal[V any](resp []byte, err error) (*V, error) {
+	if err != nil {
+		return nil, err
+	}
+	var val V
+	if err := proto.Unmarshal(resp, any(&val).(proto.Message)); err != nil {
+		return nil, errs.WrapMsg(err, "local cache proto.Unmarshal error")
+	}
+	return &val, nil
+}
+
+type cacheProto[V any] struct{}
+
+func (cacheProto[V]) Marshal(resp *V, err error) ([]byte, error) {
+	if err != nil {
+		return nil, err
+	}
+	return proto.Marshal(any(resp).(proto.Message))
+}
+
+func (cacheProto[V]) Unmarshal(resp []byte, err error) (*V, error) {
+	if err != nil {
+		return nil, err
+	}
+	var val V
+	if err := proto.Unmarshal(resp, any(&val).(proto.Message)); err != nil {
+		return nil, errs.WrapMsg(err, "local cache proto.Unmarshal error")
+	}
+	return &val, nil
+}

pkg/rpccache/conversation.go

@@ -23,6 +23,7 @@ import (
 	pbconversation "github.com/openimsdk/protocol/conversation"
 	"github.com/openimsdk/tools/errs"
 	"github.com/openimsdk/tools/log"
+	"github.com/openimsdk/tools/utils/datautil"
 	"github.com/redis/go-redis/v9"
 	"golang.org/x/sync/errgroup"
 )
@@ -36,7 +37,7 @@ func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCach
 	log.ZDebug(context.Background(), "ConversationLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable())
 	x := &ConversationLocalCache{
 		client: client,
-		local: localcache.New[any](
+		local: localcache.New[[]byte](
 			localcache.WithLocalSlotNum(lc.SlotNum),
 			localcache.WithLocalSlotSize(lc.SlotSize),
 			localcache.WithLinkSlotNum(lc.SlotNum),
@@ -52,36 +53,46 @@ func NewConversationLocalCache(client rpcclient.ConversationRpcClient, localCach
 
 type ConversationLocalCache struct {
 	client rpcclient.ConversationRpcClient
-	local  localcache.Cache[any]
+	local  localcache.Cache[[]byte]
 }
 
 func (c *ConversationLocalCache) GetConversationIDs(ctx context.Context, ownerUserID string) (val []string, err error) {
-	log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs req", "ownerUserID", ownerUserID)
+	resp, err := c.getConversationIDs(ctx, ownerUserID)
+	if err != nil {
+		return nil, err
+	}
+	return resp.ConversationIDs, nil
+}
+
+func (c *ConversationLocalCache) getConversationIDs(ctx context.Context, ownerUserID string) (val *pbconversation.GetConversationIDsResp, err error) {
+	log.ZDebug(ctx, "ConversationLocalCache getConversationIDs req", "ownerUserID", ownerUserID)
 	defer func() {
 		if err == nil {
-			log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs return", "value", val)
+			log.ZDebug(ctx, "ConversationLocalCache getConversationIDs return", "ownerUserID", ownerUserID, "value", val)
 		} else {
-			log.ZError(ctx, "ConversationLocalCache GetConversationIDs return", err)
+			log.ZError(ctx, "ConversationLocalCache getConversationIDs return", err, "ownerUserID", ownerUserID)
 		}
 	}()
-	return localcache.AnyValue[[]string](c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) (any, error) {
-		log.ZDebug(ctx, "ConversationLocalCache GetConversationIDs rpc", "ownerUserID", ownerUserID)
-		return c.client.GetConversationIDs(ctx, ownerUserID)
+	var cache cacheProto[pbconversation.GetConversationIDsResp]
+	return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationIDsKey(ownerUserID), func(ctx context.Context) ([]byte, error) {
+		log.ZDebug(ctx, "ConversationLocalCache getConversationIDs rpc", "ownerUserID", ownerUserID)
+		return cache.Marshal(c.client.Client.GetConversationIDs(ctx, &pbconversation.GetConversationIDsReq{UserID: ownerUserID}))
 	}))
 }
 
 func (c *ConversationLocalCache) GetConversation(ctx context.Context, userID, conversationID string) (val *pbconversation.Conversation, err error) {
 	log.ZDebug(ctx, "ConversationLocalCache GetConversation req", "userID", userID, "conversationID", conversationID)
 	defer func() {
 		if err == nil {
-			log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "value", val)
+			log.ZDebug(ctx, "ConversationLocalCache GetConversation return", "userID", userID, "conversationID", conversationID, "value", val)
 		} else {
-			log.ZError(ctx, "ConversationLocalCache GetConversation return", err)
+			log.ZError(ctx, "ConversationLocalCache GetConversation return", err, "userID", userID, "conversationID", conversationID)
 		}
 	}()
-	return localcache.AnyValue[*pbconversation.Conversation](c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) (any, error) {
+	var cache cacheProto[pbconversation.Conversation]
+	return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationKey(userID, conversationID), func(ctx context.Context) ([]byte, error) {
 		log.ZDebug(ctx, "ConversationLocalCache GetConversation rpc", "userID", userID, "conversationID", conversationID)
-		return c.client.GetConversation(ctx, userID, conversationID)
+		return cache.Marshal(c.client.GetConversation(ctx, userID, conversationID))
 	}))
 }
 
@@ -126,9 +137,19 @@ func (c *ConversationLocalCache) GetConversations(ctx context.Context, ownerUser
 	return conversations, nil
 }
 
-func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (*listMap[string], error) {
-	return localcache.AnyValue[*listMap[string]](c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) (any, error) {
-		return newListMap(c.client.GetConversationNotReceiveMessageUserIDs(ctx, conversationID))
+func (c *ConversationLocalCache) getConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) (val *pbconversation.GetConversationNotReceiveMessageUserIDsResp, err error) {
+	log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs req", "conversationID", conversationID)
+	defer func() {
+		if err == nil {
+			log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", "conversationID", conversationID, "value", val)
+		} else {
+			log.ZError(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs return", err, "conversationID", conversationID)
+		}
+	}()
+	var cache cacheProto[pbconversation.GetConversationNotReceiveMessageUserIDsResp]
+	return cache.Unmarshal(c.local.Get(ctx, cachekey.GetConversationNotReceiveMessageUserIDsKey(conversationID), func(ctx context.Context) ([]byte, error) {
+		log.ZDebug(ctx, "ConversationLocalCache getConversationNotReceiveMessageUserIDs rpc", "conversationID", conversationID)
+		return cache.Marshal(c.client.Client.GetConversationNotReceiveMessageUserIDs(ctx, &pbconversation.GetConversationNotReceiveMessageUserIDsReq{ConversationID: conversationID}))
 	}))
 }
 
@@ -137,13 +158,13 @@ func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDs(ctx con
 	if err != nil {
 		return nil, err
 	}
-	return res.List, nil
+	return res.UserIDs, nil
 }
 
 func (c *ConversationLocalCache) GetConversationNotReceiveMessageUserIDMap(ctx context.Context, conversationID string) (map[string]struct{}, error) {
 	res, err := c.getConversationNotReceiveMessageUserIDs(ctx, conversationID)
 	if err != nil {
 		return nil, err
 	}
-	return res.Map, nil
+	return datautil.SliceSet(res.UserIDs), nil
 }

pkg/rpccache/friend.go

@@ -16,7 +16,8 @@ package rpccache
 
 import (
 	"context"
-	cachekey2 "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey"
+	"github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/cachekey"
+	"github.com/openimsdk/protocol/relation"
 
 	"github.com/openimsdk/open-im-server/v3/pkg/common/config"
 	"github.com/openimsdk/open-im-server/v3/pkg/localcache"
@@ -30,7 +31,7 @@ func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.Lo
 	log.ZDebug(context.Background(), "FriendLocalCache", "topic", lc.Topic, "slotNum", lc.SlotNum, "slotSize", lc.SlotSize, "enable", lc.Enable())
 	x := &FriendLocalCache{
 		client: client,
-		local: localcache.New[any](
+		local: localcache.New[[]byte](
 			localcache.WithLocalSlotNum(lc.SlotNum),
 			localcache.WithLocalSlotSize(lc.SlotSize),
 			localcache.WithLinkSlotNum(lc.SlotNum),
@@ -46,36 +47,55 @@ func NewFriendLocalCache(client rpcclient.FriendRpcClient, localCache *config.Lo
 
 type FriendLocalCache struct {
 	client rpcclient.FriendRpcClient
-	local  localcache.Cache[any]
+	local  localcache.Cache[[]byte]
 }
 
 func (f *FriendLocalCache) IsFriend(ctx context.Context, possibleFriendUserID, userID string) (val bool, err error) {
-	log.ZDebug(ctx, "FriendLocalCache IsFriend req", "possibleFriendUserID", possibleFriendUserID, "userID", userID)
+	res, err := f.isFriend(ctx, possibleFriendUserID, userID)
+	if err != nil {
+		return false, err
+	}
+	return res.InUser1Friends, nil
+}
+
+func (f *FriendLocalCache) isFriend(ctx context.Context, possibleFriendUserID, userID string) (val *relation.IsFriendResp, err error) {
+	log.ZDebug(ctx, "FriendLocalCache isFriend req", "possibleFriendUserID", possibleFriendUserID, "userID", userID)
 	defer func() {
 		if err == nil {
-			log.ZDebug(ctx, "FriendLocalCache IsFriend return", "value", val)
+			log.ZDebug(ctx, "FriendLocalCache isFriend return", "possibleFriendUserID", possibleFriendUserID, "userID", userID, "value", val)
 		} else {
-			log.ZError(ctx, "FriendLocalCache IsFriend return", err)
+			log.ZError(ctx, "FriendLocalCache isFriend return", err, "possibleFriendUserID", possibleFriendUserID, "userID", userID)
 		}
 	}()
-	return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey2.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) (any, error) {
-		log.ZDebug(ctx, "FriendLocalCache IsFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID)
-		return f.client.IsFriend(ctx, possibleFriendUserID, userID)
-	}, cachekey2.GetFriendIDsKey(possibleFriendUserID)))
+	var cache cacheProto[relation.IsFriendResp]
+	return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsFriendKey(possibleFriendUserID, userID), func(ctx context.Context) ([]byte, error) {
+		log.ZDebug(ctx, "FriendLocalCache isFriend rpc", "possibleFriendUserID", possibleFriendUserID, "userID", userID)
+		return cache.Marshal(f.client.Client.IsFriend(ctx, &relation.IsFriendReq{UserID1: userID, UserID2: possibleFriendUserID}))
+	}, cachekey.GetFriendIDsKey(possibleFriendUserID)))
 }
 
 // IsBlack possibleBlackUserID selfUserID.
 func (f *FriendLocalCache) IsBlack(ctx context.Context, possibleBlackUserID, userID string) (val bool, err error) {
-	log.ZDebug(ctx, "FriendLocalCache IsBlack req", "possibleBlackUserID", possibleBlackUserID, "userID", userID)
+	res, err := f.isBlack(ctx, possibleBlackUserID, userID)
+	if err != nil {
+		return false, err
+	}
+	return res.InUser2Blacks, nil
+}
+
+// IsBlack possibleBlackUserID selfUserID.
+func (f *FriendLocalCache) isBlack(ctx context.Context, possibleBlackUserID, userID string) (val *relation.IsBlackResp, err error) {
+	log.ZDebug(ctx, "FriendLocalCache isBlack req", "possibleBlackUserID", possibleBlackUserID, "userID", userID)
 	defer func() {
 		if err == nil {
-			log.ZDebug(ctx, "FriendLocalCache IsBlack return", "value", val)
+			log.ZDebug(ctx, "FriendLocalCache isBlack return", "possibleBlackUserID", possibleBlackUserID, "userID", userID, "value", val)
 		} else {
-			log.ZError(ctx, "FriendLocalCache IsBlack return", err)
+			log.ZError(ctx, "FriendLocalCache isBlack return", err, "possibleBlackUserID", possibleBlackUserID, "userID", userID)
 		}
 	}()
-	return localcache.AnyValue[bool](f.local.GetLink(ctx, cachekey2.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) (any, error) {
+	var cache cacheProto[relation.IsBlackResp]
+	return cache.Unmarshal(f.local.GetLink(ctx, cachekey.GetIsBlackIDsKey(possibleBlackUserID, userID), func(ctx context.Context) ([]byte, error) {
 		log.ZDebug(ctx, "FriendLocalCache IsBlack rpc", "possibleBlackUserID", possibleBlackUserID, "userID", userID)
-		return f.client.IsBlack(ctx, possibleBlackUserID, userID)
-	}, cachekey2.GetBlackIDsKey(userID)))
+		return cache.Marshal(f.client.Client.IsBlack(ctx, &relation.IsBlackReq{UserID1: possibleBlackUserID, UserID2: userID}))
+	}, cachekey.GetBlackIDsKey(userID)))
 }