From f95a6fb8df667074b99c46d82e0b8b62c225b321 Mon Sep 17 00:00:00 2001 From: kavos Date: Wed, 21 Aug 2024 17:52:51 +0900 Subject: [PATCH 1/3] fix sodan --- src/scraper/messages.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scraper/messages.go b/src/scraper/messages.go index be02f26..2a05742 100644 --- a/src/scraper/messages.go +++ b/src/scraper/messages.go @@ -13,7 +13,7 @@ func (s *Scraper) GetSodanMessages(bot *traqwsbot.Bot) { API(). MessageApi. GetMessages(context.Background(), "aff37b5f-0911-4255-81c3-b49985c8943f"). - Offset(int32(15)). + Offset(int32(16)). Limit(int32(20)). Execute() if err != nil { @@ -65,7 +65,7 @@ func (s *Scraper) GetSodanMessages(bot *traqwsbot.Bot) { } s.GetSodanSubMessages(bot, "98ea48da-64e8-4f69-9d0d-80690b682670", 11, 52) - s.GetSodanSubMessages(bot, "30c30aa5-c380-4324-b227-0ca85c34801c", 0, 32) + s.GetSodanSubMessages(bot, "30c30aa5-c380-4324-b227-0ca85c34801c", 4, 32) s.GetSodanSubMessages(bot, "7ec94f1d-1920-4e15-bfc5-049c9a289692", 5, 18) s.GetSodanSubMessages(bot, "c67abb48-3fb0-4486-98ad-4b6947998ad5", 0, 21) s.GetSodanSubMessages(bot, "eb5a0035-a340-4cf6-a9e0-94ddfabe9337", 0, 2) From 911f70a0c8913f704dc2aebbda2d442bb1b3bd1b Mon Sep 17 00:00:00 2001 From: bn256 Date: Wed, 21 Aug 2024 18:28:45 +0900 Subject: [PATCH 2/3] fixed around tags --- src/handler/handler.go | 30 ++++++++++++++---------------- src/handler/model.go | 3 ++- src/scraper/messages.go | 9 +++++---- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/handler/handler.go b/src/handler/handler.go index 01b424f..a8241f1 100644 --- a/src/handler/handler.go +++ b/src/handler/handler.go @@ -176,22 +176,20 @@ func (h *Handler) GetSodanHandler(c echo.Context) error { } Response.Title = wikiContent.Name - // tagの実装がまだなのでコメントアウト,直下にダミーを実装 - // var tags []Tag_fromDB - // var howManyTags int - // err = h.db.Get(&tags, "select * from tags join tags_in_wiki on tag_id where wiki_id = ?", wikiId) - // if err != nil { - // if errors.Is(err, sql.ErrNoRows) { - // return c.NoContent(http.StatusNotFound) - // } - // log.Printf("failed to get tags: %s\n", err) - // return c.NoContent(http.StatusInternalServerError) - // } - // howManyTags = len(tags) - // for i := 0; i < howManyTags; i++ { - // Response.Tags = append(Response.Tags, tags[i].TagName) - // } - Response.Tags = []string{"hoge", "fuga"} + var tags []Tag_fromDB + var howManyTags int + err = h.db.Select(&tags, "select * from tags where wiki_id = ?", wikiId) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return c.NoContent(http.StatusNotFound) + } + log.Printf("failed to get tags: %s\n", err) + return c.NoContent(http.StatusInternalServerError) + } + howManyTags = len(tags) + for i := 0; i < howManyTags; i++ { + Response.Tags = append(Response.Tags, tags[i].TagName) + } var messageContents []SodanContent_fromDB var howManyMessages int diff --git a/src/handler/model.go b/src/handler/model.go index dd5a234..93ba473 100644 --- a/src/handler/model.go +++ b/src/handler/model.go @@ -46,9 +46,10 @@ type WikiContent_fromDB struct { // sqlよりtagを取ってくるときに使う type Tag_fromDB struct { + TagID int `db:"id"` WikiID int `db:"wiki_id"` - TagID int `db:"tag_id"` TagName string `db:"name"` + TagScore float64 `db:"tag_score"` } // sqlのmessagesから情報を取ってくるときに使う diff --git a/src/scraper/messages.go b/src/scraper/messages.go index be02f26..ea7a773 100644 --- a/src/scraper/messages.go +++ b/src/scraper/messages.go @@ -2,10 +2,11 @@ package scraper import ( "context" - "github.com/traPtitech/go-traq" - traqwsbot "github.com/traPtitech/traq-ws-bot" "log" "regexp" + + "github.com/traPtitech/go-traq" + traqwsbot "github.com/traPtitech/traq-ws-bot" ) func (s *Scraper) GetSodanMessages(bot *traqwsbot.Bot) { @@ -13,7 +14,7 @@ func (s *Scraper) GetSodanMessages(bot *traqwsbot.Bot) { API(). MessageApi. GetMessages(context.Background(), "aff37b5f-0911-4255-81c3-b49985c8943f"). - Offset(int32(15)). + Offset(int32(16)). Limit(int32(20)). Execute() if err != nil { @@ -65,7 +66,7 @@ func (s *Scraper) GetSodanMessages(bot *traqwsbot.Bot) { } s.GetSodanSubMessages(bot, "98ea48da-64e8-4f69-9d0d-80690b682670", 11, 52) - s.GetSodanSubMessages(bot, "30c30aa5-c380-4324-b227-0ca85c34801c", 0, 32) + s.GetSodanSubMessages(bot, "30c30aa5-c380-4324-b227-0ca85c34801c", 6, 32) s.GetSodanSubMessages(bot, "7ec94f1d-1920-4e15-bfc5-049c9a289692", 5, 18) s.GetSodanSubMessages(bot, "c67abb48-3fb0-4486-98ad-4b6947998ad5", 0, 21) s.GetSodanSubMessages(bot, "eb5a0035-a340-4cf6-a9e0-94ddfabe9337", 0, 2) From 3b292cceff27b9ce8fc56edbfccc366b053a2144 Mon Sep 17 00:00:00 2001 From: kavos Date: Wed, 21 Aug 2024 18:41:47 +0900 Subject: [PATCH 3/3] fixed --- src/scraper/messages.go | 3 ++- src/search/index.go | 9 +++++++++ src/search/search.go | 12 +++++++++--- test/blevesearch/main.go | 13 +++++++++---- test/db.http | 3 +++ 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/scraper/messages.go b/src/scraper/messages.go index 2a05742..644d419 100644 --- a/src/scraper/messages.go +++ b/src/scraper/messages.go @@ -65,13 +65,14 @@ func (s *Scraper) GetSodanMessages(bot *traqwsbot.Bot) { } s.GetSodanSubMessages(bot, "98ea48da-64e8-4f69-9d0d-80690b682670", 11, 52) - s.GetSodanSubMessages(bot, "30c30aa5-c380-4324-b227-0ca85c34801c", 4, 32) + s.GetSodanSubMessages(bot, "30c30aa5-c380-4324-b227-0ca85c34801c", 6, 32) s.GetSodanSubMessages(bot, "7ec94f1d-1920-4e15-bfc5-049c9a289692", 5, 18) s.GetSodanSubMessages(bot, "c67abb48-3fb0-4486-98ad-4b6947998ad5", 0, 21) s.GetSodanSubMessages(bot, "eb5a0035-a340-4cf6-a9e0-94ddfabe9337", 0, 2) s.updateWikisContent() s.setSodanTags() + s.setIndexing() } func (s *Scraper) GetSodanSubMessages(bot *traqwsbot.Bot, channelId string, offset int, limit int) { diff --git a/src/search/index.go b/src/search/index.go index dbc36a4..81ec9a7 100644 --- a/src/search/index.go +++ b/src/search/index.go @@ -92,4 +92,13 @@ func Indexing(data []IndexData) { return } log.Printf("[From search engine] Finish index successfully. doc count: %d\n\n", docCount) + + err = index.Close() + if err != nil { + log.Printf("[Error from search engine] failed to close index: %v\n", err) + return + } + + res := Search("windows", 10, 0) + log.Println(res) } diff --git a/src/search/search.go b/src/search/search.go index c288ef2..59fbbe3 100644 --- a/src/search/search.go +++ b/src/search/search.go @@ -10,13 +10,14 @@ import ( // return array of ID // return empty array if no result or error func Search(query string, limit int, offset int) []int { + log.Println("[Info from search] Searching by query: ", query) if _, err := os.Stat("index.bleve"); err != nil { log.Println("[Error from search] Index file does not exist. Please create index file first.") } index, err := bleve.Open("index.bleve") if err != nil { - log.Printf("failed to open index: %v\n", err) + log.Printf("[Error from search] failed to open index: %v\n", err) } bleveQuery := bleve.NewMatchQuery(query) @@ -24,16 +25,21 @@ func Search(query string, limit int, offset int) []int { search := bleve.NewSearchRequest(bleveQuery) searchResults, err := index.Search(search) if err != nil { - log.Printf("failed to search by query\"%s\": %v\n", query, err) + log.Printf("[Error from search] failed to search by query\"%s\": %v\n", query, err) return []int{} } + err = index.Close() + if err != nil { + log.Printf("[Error from search] failed to close index: %v\n", err) + } + returnCount := min(limit, int(searchResults.Total)-offset) res := make([]int, returnCount) for i := offset; i < offset+returnCount; i++ { res[i-offset], err = strconv.Atoi(searchResults.Hits[i].ID) if err != nil { - log.Printf("failed to convert ID to int: %v\n", err) + log.Printf("[Error from search] failed to convert ID to int: %v\n", err) return []int{} } } diff --git a/test/blevesearch/main.go b/test/blevesearch/main.go index caae40a..5a28462 100644 --- a/test/blevesearch/main.go +++ b/test/blevesearch/main.go @@ -17,7 +17,12 @@ type IndexData struct { } func main() { - index := setting() + setting() + index, err := bleve.Open("index.bleve") + if err != nil { + log.Println(err) + return + } // search for some text query := bleve.NewMatchQuery("作る 並行") @@ -32,9 +37,10 @@ func main() { for _, hit := range searchResults.Hits { log.Printf("ID: %s, Score: %f\n", hit.ID, hit.Score) } + index.Close() } -func setting() bleve.Index { +func setting() { // open a new index typeMapping := bleve.NewTextFieldMapping() typeMapping.Analyzer = keyword.Name @@ -130,6 +136,5 @@ func setting() bleve.Index { panic(err) } log.Printf("doc count: %d\n\n", docCount) - - return index + index.Close() } diff --git a/test/db.http b/test/db.http index 2cb8298..06a85cc 100644 --- a/test/db.http +++ b/test/db.http @@ -22,3 +22,6 @@ Content-Type: application/json { "name": "test" } + +### +GET http://localhost:8080/search \ No newline at end of file