From 2a2893c30ea2659c31af0846190aed98275d272a Mon Sep 17 00:00:00 2001 From: mixinju Date: Wed, 10 Aug 2022 16:36:35 +0800 Subject: [PATCH] =?UTF-8?q?Go=E5=8E=9F=E7=94=9F=20SDK=20=E7=B2=97=E7=95=A5?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/parser.go | 2 +- main.go | 4 +- ...76\350\256\241\346\214\207\345\215\227.md" | 7 ++ sdk/base.go | 33 ++++++++ sdk/client.go | 82 +++++++++++++++++++ sdk/database.go | 32 ++++++++ sdk/index.go | 43 ++++++++++ sdk/word.go | 15 ++++ searcher/container.go | 1 - searcher/storage/leveldb_storage.go | 7 +- searcher/words/tokenizer.go | 3 +- web/service/service.go | 4 +- 12 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 "sdk/SDK \350\256\276\350\256\241\346\214\207\345\215\227.md" create mode 100644 sdk/base.go create mode 100644 sdk/client.go create mode 100644 sdk/database.go create mode 100644 sdk/index.go create mode 100644 sdk/word.go diff --git a/core/parser.go b/core/parser.go index 9b6dd54..78404ed 100644 --- a/core/parser.go +++ b/core/parser.go @@ -13,7 +13,7 @@ import ( // Parser 解析器 func Parser() *global.Config { - var addr = flag.String("addr", "0.0.0.0:5678", "设置监听地址和端口") + var addr = flag.String("addr", "127.0.0.1:5678", "设置监听地址和端口") //兼容windows dir := fmt.Sprintf(".%sdata", string(os.PathSeparator)) diff --git a/main.go b/main.go index 07c3538..64231fd 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,8 @@ package main -import "gofound/core" +import ( + "gofound/core" +) func main() { //初始化容器和参数解析 diff --git "a/sdk/SDK \350\256\276\350\256\241\346\214\207\345\215\227.md" "b/sdk/SDK \350\256\276\350\256\241\346\214\207\345\215\227.md" new file mode 100644 index 0000000..43011c4 --- /dev/null +++ "b/sdk/SDK \350\256\276\350\256\241\346\214\207\345\215\227.md" @@ -0,0 +1,7 @@ +# GoFound SDK设计指南 + +## 支持自定义配置 +在支持自定义配置的时候，同时提供默认配置项 +## +支持`gofound` 提供的所有操作,增删改查等 + diff --git a/sdk/base.go b/sdk/base.go new file mode 100644 index 0000000..953bd2a --- /dev/null +++ b/sdk/base.go @@ -0,0 +1,33 @@ +package gofound + +import ( + "gofound/searcher/model" + "gofound/searcher/system" + "runtime" +) + +// Query 查询 +func (c *Client) Query(req *model.SearchRequest) (*model.SearchResult, error) { + r, err := c.container.GetDataBase(req.Database).MultiSearch(req) + if err != nil { + return nil, err + } + + return r, nil +} + +func (*Client) GC() { + runtime.GC() +} +func (c *Client) Status() (map[string]interface{}, error) { + var m runtime.MemStats + runtime.ReadMemStats(&m) + + // TODO 其他系统信息 + r := map[string]interface{}{ + "memory": system.GetMemStat(), + "cpu": system.GetCPUStatus(), + "disk": system.GetDiskStat(), + } + return r, nil +} diff --git a/sdk/client.go b/sdk/client.go new file mode 100644 index 0000000..1c4b6df --- /dev/null +++ b/sdk/client.go @@ -0,0 +1,82 @@ +package gofound + +import ( + "fmt" + "gofound/core" + "gofound/global" + "gofound/searcher" + "os" + "runtime" + "sync" +) + +var once sync.Once + +// Client 应该对外部屏蔽细节 +// 尽量少的提供接口，但是又要保证功能性 +type Client struct { + config *global.Config //服务配置 + container *searcher.Container //运行实体 +} + +func newDefaultConfig() *global.Config { + return &global.Config{ + Addr: "127.0.0.1:5678", + Data: fmt.Sprintf(".%sdata", string(os.PathSeparator)), + Debug: true, + Dictionary: "./data/dictionary.txt", + EnableAdmin: true, + Gomaxprocs: runtime.NumCPU() * 2, + Shard: 0, + Auth: "", + EnableGzip: true, + Timeout: 10 * 60, + } +} +func newTokenizerAndContainer(config *global.Config) *searcher.Container { + tokenizer := core.NewTokenizer(global.CONFIG.Dictionary) + return core.NewContainer(tokenizer) +} + +// NewClient 通过参数进行配置,必须指定全部参数 +func NewClient(config *global.Config) *Client { + global.CONFIG = config + //初始化分词器 + container := newTokenizerAndContainer(config) + global.Container = container + return &Client{ + config: config, + container: container, + } +} + +// Default 使用默认参数创建服务 +func Default() *Client { + global.CONFIG = newDefaultConfig() + container := newTokenizerAndContainer(global.CONFIG) + global.Container = container + return &Client{ + config: global.CONFIG, + container: container, + } +} + +// SetAddr 设置Web服务地址 +func (c *Client) SetAddr(addr string) *Client { + if addr == "" { + return c + } + c.config.Addr = addr + return c +} + +// SetData 设置数据存放地址 +func (c *Client) SetData(dir string) *Client { + if dir == "" { + return c + } + c.config.Data = dir + return c +} + +//TODO 其他配置项 diff --git a/sdk/database.go b/sdk/database.go new file mode 100644 index 0000000..f80502f --- /dev/null +++ b/sdk/database.go @@ -0,0 +1,32 @@ +package gofound + +import ( + "gofound/searcher" + + "github.com/syndtr/goleveldb/leveldb/errors" +) + +// Show 查看数据库 +func (c *Client) Show() (map[string]*searcher.Engine, error) { + // 保持分格一致 + return c.container.GetDataBases(), nil +} + +// Drop 删除数据库 +func (c *Client) Drop(dbName string) error { + if dbName == "" { + return errors.New("database not exist") + } + if err := c.container.DropDataBase(dbName); err != nil { + return err + } + return nil +} + +// Create 创建数据库 +func (c *Client) Create(dbName string) (*searcher.Engine, error) { + if dbName == "" { + return nil, errors.New("database name is empty") + } + return c.container.GetDataBase(dbName), nil +} diff --git a/sdk/index.go b/sdk/index.go new file mode 100644 index 0000000..912b452 --- /dev/null +++ b/sdk/index.go @@ -0,0 +1,43 @@ +package gofound + +import ( + "errors" + "gofound/searcher/model" +) + +// AddIndex 添加索引 +func (c *Client) AddIndex(dbName string, request *model.IndexDoc) error { + if request.Text == "" { + return errors.New("text is empty") + } + c.container.GetDataBase(dbName).IndexDocument(request) + + return nil +} + +// BatchAddIndex 批次添加索引 +func (c *Client) BatchAddIndex(dbName string, documents []*model.IndexDoc) error { + db := c.container.GetDataBase(dbName) + // 数据预处理 + for _, doc := range documents { + if doc.Text == "" { + return errors.New("text is empty") + } + if doc.Document == nil { + return errors.New("document is empty") + } + } + for _, doc := range documents { + go db.IndexDocument(doc) + } + return nil +} + +// RemoveIndex 删除索引 +func (c *Client) RemoveIndex(dbName string, data *model.RemoveIndexModel) error { + db := c.container.GetDataBase(dbName) + if err := db.RemoveIndex(data.Id); err != nil { + return err + } + return nil +} diff --git a/sdk/word.go b/sdk/word.go new file mode 100644 index 0000000..788c775 --- /dev/null +++ b/sdk/word.go @@ -0,0 +1,15 @@ +package gofound + +// WordCut 分词 +func (c *Client) WordCut(keyword string) []string { + return c.container.Tokenizer.Cut(keyword) +} + +// BatchWordCut 批量分词 +func (c *Client) BatchWordCut(keywords []string) *[][]string { + res := make([][]string, len(keywords)) + for _, w := range keywords { + res = append(res, c.container.Tokenizer.Cut(w)) + } + return &res +} diff --git a/searcher/container.go b/searcher/container.go index 679ff10..9ef360a 100644 --- a/searcher/container.go +++ b/searcher/container.go @@ -78,7 +78,6 @@ func (c *Container) GetDataBase(name string) *Engine { engine = c.NewEngine(name) c.engines[name] = engine //释放引擎 - } return engine diff --git a/searcher/storage/leveldb_storage.go b/searcher/storage/leveldb_storage.go index e5ebd09..5bcadfd 100644 --- a/searcher/storage/leveldb_storage.go +++ b/searcher/storage/leveldb_storage.go @@ -1,12 +1,13 @@ package storage import ( - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/opt" "log" "sync" "time" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/filter" + "github.com/syndtr/goleveldb/leveldb/opt" ) // LeveldbStorage TODO 要支持事务 diff --git a/searcher/words/tokenizer.go b/searcher/words/tokenizer.go index e5cc24e..6d948ad 100644 --- a/searcher/words/tokenizer.go +++ b/searcher/words/tokenizer.go @@ -2,9 +2,10 @@ package words import ( "embed" - "github.com/wangbin/jiebago" "gofound/searcher/utils" "strings" + + "github.com/wangbin/jiebago" ) var ( diff --git a/web/service/service.go b/web/service/service.go index c98c57a..673d7fd 100644 --- a/web/service/service.go +++ b/web/service/service.go @@ -23,8 +23,8 @@ func Callback() map[string]interface{} { "dataSize": system.GetFloat64MB(utils.DirSizeB(global.CONFIG.Data)), "executable": os.Args[0], "dbs": global.Container.GetDataBaseNumber(), - //"indexCount": global.Container.GetIndexCount(), - //"documentCount": global.Container.GetDocumentCount(), + //"indexCount": global.container.GetIndexCount(), + //"documentCount": global.container.GetDocumentCount(), "pid": os.Getpid(), "enableAuth": global.CONFIG.Auth != "", "enableGzip": global.CONFIG.EnableGzip,