From b5e911189cd69a536035543281f198befaf80873 Mon Sep 17 00:00:00 2001 From: liubo <742173262@qq.com> Date: Thu, 5 Mar 2026 21:46:08 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E5=9C=A8=E5=87=BD=E6=95=B0=E5=BC=80?= =?UTF-8?q?=E5=A4=B4=E6=B7=BB=E5=8A=A0=E8=BE=93=E5=85=A5=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E6=A3=80=E6=B5=8B=EF=BC=8C=E5=BD=93=E7=84=A6=E7=82=B9=E5=9C=A8?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E6=A1=86=E3=80=81textarea=E3=80=81select=20?= =?UTF-8?q?=E6=88=96=E5=8F=AF=E7=BC=96=E8=BE=91=E5=85=83=E7=B4=A0=E4=B8=AD?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E7=9B=B4=E6=8E=A5=E8=BF=94=E5=9B=9E=E4=B8=8D?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E5=BF=AB=E6=8D=B7=E9=94=AE=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- static/js/kancloud.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/static/js/kancloud.js b/static/js/kancloud.js index f759cd85d..810a27682 100644 --- a/static/js/kancloud.js +++ b/static/js/kancloud.js @@ -242,6 +242,16 @@ function initHighlighting() { } function handleEvent(event) { + // 如果焦点在输入框、textarea或可编辑元素中，不执行快捷键操作 + var target = event.target; + var tagName = target.tagName.toLowerCase(); + var isInputElement = tagName === 'input' || tagName === 'textarea' || tagName === 'select'; + var isContentEditable = target.isContentEditable || target.contentEditable === 'true'; + + if (isInputElement || isContentEditable) { + return; + } + switch (event.keyCode) { case 70: // ctrl + f 打开搜索面板 并获取焦点 $(".navg-item[data-mode='search']").click(); From 661af2e7419a6bfd258fa0a0f826326df320460e Mon Sep 17 00:00:00 2001 From: liubo <742173262@qq.com> Date: Fri, 20 Mar 2026 00:08:01 +0800 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=20Cherry=20Markd?= =?UTF-8?q?own=20=E9=98=85=E8=AF=BB=E9=A1=B5=E7=9B=AE=E5=BD=95=E6=BB=9A?= =?UTF-8?q?=E5=8A=A8=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 为 Cherry 编辑器阅读模式的目录添加最大高度限制 - 当目录过长时支持垂直滚动查看后续条目 - 使用 calc(100vh - 180px) 动态计算合适的滚动容器高度 - 启用 webkit 平滑滚动以改善移动端体验 --- views/document/cherry_read.tpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/views/document/cherry_read.tpl b/views/document/cherry_read.tpl index d134bfc17..b16cad044 100644 --- a/views/document/cherry_read.tpl +++ b/views/document/cherry_read.tpl @@ -50,6 +50,13 @@ display: none; } } + + .m-manual.manual-reader .manual-article.cherry-markdown .article-body .toc { + max-height: calc(100vh - 180px); + overflow-y: auto; + overflow-x: hidden; + -webkit-overflow-scrolling: touch; + } From 0e8476b15b3beee3eeeb9dee6f524f2fc0156347 Mon Sep 17 00:00:00 2001 From: liubo <742173262@qq.com> Date: Fri, 20 Mar 2026 19:38:26 +0800 Subject: [PATCH 3/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dmindoc=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E5=9C=A8=E5=AE=89=E5=8D=93termux=E4=B8=8A=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E5=8C=BA=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 6 ++++++ utils/android_time.go | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 utils/android_time.go diff --git a/main.go b/main.go index 0edfe0162..82ee642ec 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ import ( "github.com/mindoc-org/mindoc/commands" "github.com/mindoc-org/mindoc/commands/daemon" _ "github.com/mindoc-org/mindoc/routers" + "github.com/mindoc-org/mindoc/utils" ) func isViaDaemonUnix() bool { @@ -49,6 +50,11 @@ func main() { d := daemon.NewDaemon() + // 安卓 go/src/time/zoneinfo_android.go 固定localLoc 为 UTC + if runtime.GOOS == "android" { + utils.FixTimezone() + } + if runtime.GOOS != "windows" && !isViaDaemonUnix() { s, err := service.New(d, d.Config()) diff --git a/utils/android_time.go b/utils/android_time.go new file mode 100644 index 000000000..f1b4ed2eb --- /dev/null +++ b/utils/android_time.go @@ -0,0 +1,19 @@ +package utils + +import ( + "os/exec" + "strings" + "time" +) + +func FixTimezone() { + out, err := exec.Command("/system/bin/getprop", "persist.sys.timezone").Output() + if err != nil { + return + } + timeZone, err := time.LoadLocation(strings.TrimSpace(string(out))) + if err != nil { + return + } + time.Local = timeZone +} From 5ebfd66e6bda47458722460070cedd16bb2ec52c Mon Sep 17 00:00:00 2001 From: liubo <742173262@qq.com> Date: Mon, 23 Mar 2026 22:46:53 +0800 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E5=85=A8?= =?UTF-8?q?=E5=B1=80=E6=90=9C=E7=B4=A2=E5=8A=9F=E8=83=BD=20-=20=E6=8F=90?= =?UTF-8?q?=E5=8D=87=E6=90=9C=E7=B4=A2=E5=87=86=E7=A1=AE=E5=BA=A6=E4=B8=8E?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=E8=B4=A8=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. SQL搜索添加相关性排序：CASE WHEN标题匹配THEN 2 ELSE 1排序，标题匹配结果优先展示 2. TF-IDF算法修正：COUNT改为SUM计算总词频；每个词独立计算IDF；添加查询词覆盖率加成 3. 分词过滤优化：过滤单字符标点/特殊字符避免匹配大量无关文档；原始关键词加入搜索词列表 4. 批量加载优化：将逐条DB查询改为批量IN查询(Document/Blog/Book/Member)，大幅减少DB往返 5. 排序算法优化：冒泡排序改为sort.Slice，时间复杂度从O(n²)降到O(n log n) 6. 标题加权方式重构：移除10次标题重复拼接，改为搜索结果层1.5x boost系数，避免索引数据污染 7. 精确匹配boost(5x)：文档内容包含原始搜索关键词时大幅提分，解决专有名词排名靠后问题 8. 词长权重：长词(更具体)的TF-IDF贡献按log2(1+len)加权，专有名词匹配权重高于短词 9. 对数TF替代原始TF：公式从wordCount/totalWordsInDoc改为1+log(wordCount+1)，解决空/短文档排名异常高的问题 10. 分页时机修复：标题boost和精确匹配boost在分页截取之前完成，确保高相关性文档不被遗漏 11. 总文档数查询优化：CONCAT+DISTINCT改为子查询，可利用联合索引提升性能 12. 新增mindoc reindex CLI命令：支持全量重建倒排索引，清空旧数据后按批次重建 --- commands/command.go | 6 + controllers/SearchController.go | 308 ++++++++++++++++++++++++-------- models/Blog.go | 3 +- models/ContentReverseIndex.go | 261 +++++++++++++++++---------- models/DocumentModel.go | 3 +- models/DocumentSearchResult.go | 26 +-- utils/segmenter/segmenter.go | 6 + 7 files changed, 427 insertions(+), 186 deletions(-) diff --git a/commands/command.go b/commands/command.go index 2852d990e..b40d2b6bb 100644 --- a/commands/command.go +++ b/commands/command.go @@ -237,6 +237,12 @@ func RegisterCommand() { } else if len(os.Args) >= 2 && os.Args[1] == "update" { Update() os.Exit(0) + } else if len(os.Args) >= 2 && os.Args[1] == "reindex" { + ResolveCommand(os.Args[2:]) + fmt.Println("开始全量重建倒排索引...") + models.RebuildAllIndexes() + fmt.Println("倒排索引重建完成") + os.Exit(0) } } diff --git a/controllers/SearchController.go b/controllers/SearchController.go index 8375dd3fe..44483b564 100644 --- a/controllers/SearchController.go +++ b/controllers/SearchController.go @@ -1,9 +1,11 @@ package controllers import ( + "sort" "strconv" "strings" + "github.com/beego/beego/v2/client/orm" "github.com/beego/beego/v2/core/logs" "github.com/beego/i18n" "github.com/mindoc-org/mindoc/conf" @@ -62,16 +64,116 @@ func PerformSearchV2Raw(keyword string, pageIndex, pageSize int, memberId int) ( words = []string{keyword} } + // 将原始关键词（小写）加入搜索词列表，确保能匹配索引中存储的完整词条 + lowerKeyword := strings.ToLower(strings.TrimSpace(keyword)) + if lowerKeyword != "" { + found := false + for _, w := range words { + if w == lowerKeyword { + found = true + break + } + } + if !found { + words = append(words, lowerKeyword) + } + } + // 使用倒排索引模型进行搜索 index := models.NewContentReverseIndex() - results, totalCount, err := index.FindByWordsWithPagination(words, pageIndex, pageSize) + allResults, err := index.FindByWords(words) if err != nil { return nil, words, 0, err } + // 收集需要批量查询的ID + docIds := make([]int, 0) + blogIds := make([]int, 0) + for _, result := range allResults { + if result.ContentType == 1 { + docIds = append(docIds, result.ContentId) + } else if result.ContentType == 2 { + blogIds = append(blogIds, result.ContentId) + } + } + + // 批量加载 Document + docMap := make(map[int]*models.Document) + if len(docIds) > 0 { + var docs []*models.Document + o := orm.NewOrm() + _, err := o.QueryTable(models.NewDocument().TableNameWithPrefix()).Filter("document_id__in", docIds).All(&docs) + if err == nil { + for _, doc := range docs { + docMap[doc.DocumentId] = doc + } + } + } + + // 批量加载 Blog + blogMap := make(map[int]*models.Blog) + if len(blogIds) > 0 { + var blogs []*models.Blog + o := orm.NewOrm() + _, err := o.QueryTable(models.NewBlog().TableNameWithPrefix()).Filter("blog_id__in", blogIds).All(&blogs) + if err == nil { + for _, blog := range blogs { + blogMap[blog.BlogId] = blog + } + } + } + + // 收集需要加载的 BookId 和 MemberId + bookIds := make([]int, 0) + memberIds := make([]int, 0) + bookIdSet := make(map[int]bool) + memberIdSet := make(map[int]bool) + for _, doc := range docMap { + if doc.BookId > 0 && !bookIdSet[doc.BookId] { + bookIds = append(bookIds, doc.BookId) + bookIdSet[doc.BookId] = true + } + if doc.MemberId > 0 && !memberIdSet[doc.MemberId] { + memberIds = append(memberIds, doc.MemberId) + memberIdSet[doc.MemberId] = true + } + } + for _, blog := range blogMap { + if blog.MemberId > 0 && !memberIdSet[blog.MemberId] { + memberIds = append(memberIds, blog.MemberId) + memberIdSet[blog.MemberId] = true + } + } + + // 批量加载 Book + bookMap := make(map[int]*models.Book) + if len(bookIds) > 0 { + var books []*models.Book + o := orm.NewOrm() + _, err := o.QueryTable(models.NewBook().TableNameWithPrefix()).Filter("book_id__in", bookIds).All(&books) + if err == nil { + for _, book := range books { + bookMap[book.BookId] = book + } + } + } + + // 批量加载 Member + memberMap := make(map[int]*models.Member) + if len(memberIds) > 0 { + var members []*models.Member + o := orm.NewOrm() + _, err := o.QueryTable(models.NewMember().TableNameWithPrefix()).Filter("member_id__in", memberIds).All(&members, "member_id", "account", "real_name") + if err == nil { + for _, member := range members { + memberMap[member.MemberId] = member + } + } + } + // 构建返回结果 searchResults := make([]*SearchV2RawResult, 0) - for _, result := range results { + for _, result := range allResults { item := &SearchV2RawResult{ ContentType: result.ContentType, ContentId: result.ContentId, @@ -81,100 +183,150 @@ func PerformSearchV2Raw(keyword string, pageIndex, pageSize int, memberId int) ( // 根据内容类型获取详细信息 if result.ContentType == 1 { - // Document类型 - doc, err := models.NewDocument().Find(result.ContentId) - if err == nil { - // 检查文档权限 - book, bookErr := models.NewBook().Find(doc.BookId) - if bookErr != nil { - continue + doc, ok := docMap[result.ContentId] + if !ok { + continue + } + book, ok := bookMap[doc.BookId] + if !ok { + continue + } + + item.SearchType = "document" + item.DocumentId = doc.DocumentId + item.DocumentName = doc.DocumentName + item.BookId = doc.BookId + item.BookName = book.BookName + item.Identify = doc.Identify + item.BookIdentify = book.Identify + item.CreateTime = doc.CreateTime + item.ModifyTime = doc.ModifyTime + item.Content = doc.Release + + // 获取作者信息 + if member, ok := memberMap[doc.MemberId]; ok { + if member.RealName != "" { + item.Author = member.RealName + } else { + item.Author = member.Account } + } - item.SearchType = "document" - item.DocumentId = doc.DocumentId - item.DocumentName = doc.DocumentName - item.BookId = doc.BookId - item.BookName = book.BookName - item.Identify = doc.Identify - item.BookIdentify = book.Identify - item.CreateTime = doc.CreateTime - item.ModifyTime = doc.ModifyTime - item.Content = doc.Release - - // 获取作者信息 - if doc.MemberId > 0 { - member, _ := models.NewMember().Find(doc.MemberId, "real_name", "account") - if member != nil { - if member.RealName != "" { - item.Author = member.RealName - } else { - item.Author = member.Account - } - } + // 提取描述 + description := doc.Release + if description == "" { + description = doc.Markdown + } + description = utils.StripTags(description) + if len([]rune(description)) > 100 { + description = string([]rune(description)[:100]) + "..." + } + item.Description = description + + // 标题匹配加权：搜索词命中标题时提升分数 + titleLower := strings.ToLower(doc.DocumentName) + for _, w := range words { + if strings.Contains(titleLower, w) { + item.Score *= 1.5 } + } - // 提取描述 - description := doc.Release - if description == "" { - description = doc.Markdown + // 精确匹配加权：文档内容包含原始关键词时大幅提分 + if lowerKeyword != "" { + contentLower := strings.ToLower(utils.StripTags(doc.Release)) + if contentLower == "" { + contentLower = strings.ToLower(utils.StripTags(doc.Markdown)) } - // 去除HTML标签 - description = utils.StripTags(description) - if len([]rune(description)) > 100 { - description = string([]rune(description)[:100]) + "..." + if strings.Contains(contentLower, lowerKeyword) { + item.Score *= 5.0 } - item.Description = description - - searchResults = append(searchResults, item) } + + searchResults = append(searchResults, item) } else if result.ContentType == 2 { - // Blog类型 - blog, err := models.NewBlog().Find(result.ContentId) - if err == nil { - item.SearchType = "blog" - item.BlogId = blog.BlogId - item.BlogTitle = blog.BlogTitle - item.DocumentId = blog.BlogId - item.DocumentName = blog.BlogTitle - item.BlogIdentify = blog.BlogIdentify - item.Identify = blog.BlogIdentify - item.BlogExcerpt = blog.BlogExcerpt - item.CreateTime = blog.Created - item.ModifyTime = blog.Modified - item.Content = blog.BlogRelease - - // 获取作者信息 - if blog.MemberId > 0 { - member, _ := models.NewMember().Find(blog.MemberId, "real_name", "account") - if member != nil { - if member.RealName != "" { - item.Author = member.RealName - } else { - item.Author = member.Account - } - } + blog, ok := blogMap[result.ContentId] + if !ok { + continue + } + + item.SearchType = "blog" + item.BlogId = blog.BlogId + item.BlogTitle = blog.BlogTitle + item.DocumentId = blog.BlogId + item.DocumentName = blog.BlogTitle + item.BlogIdentify = blog.BlogIdentify + item.Identify = blog.BlogIdentify + item.BlogExcerpt = blog.BlogExcerpt + item.CreateTime = blog.Created + item.ModifyTime = blog.Modified + item.Content = blog.BlogRelease + + // 获取作者信息 + if member, ok := memberMap[blog.MemberId]; ok { + if member.RealName != "" { + item.Author = member.RealName + } else { + item.Author = member.Account } + } - // 提取描述 - description := blog.BlogExcerpt + // 提取描述 + description := blog.BlogExcerpt + if description == "" { + description = blog.BlogRelease if description == "" { - description = blog.BlogRelease - if description == "" { - description = blog.BlogContent - } + description = blog.BlogContent } - description = utils.StripTags(description) - if len([]rune(description)) > 100 { - description = string([]rune(description)[:100]) + "..." + } + description = utils.StripTags(description) + if len([]rune(description)) > 100 { + description = string([]rune(description)[:100]) + "..." + } + item.Description = description + + // 标题匹配加权：搜索词命中标题时提升分数 + titleLower := strings.ToLower(blog.BlogTitle) + for _, w := range words { + if strings.Contains(titleLower, w) { + item.Score *= 1.5 } - item.Description = description + } - searchResults = append(searchResults, item) + // 精确匹配加权：博客内容包含原始关键词时大幅提分 + if lowerKeyword != "" { + contentLower := strings.ToLower(utils.StripTags(blog.BlogRelease)) + if contentLower == "" { + contentLower = strings.ToLower(utils.StripTags(blog.BlogContent)) + } + if strings.Contains(contentLower, lowerKeyword) { + item.Score *= 5.0 + } } + + searchResults = append(searchResults, item) } } - return searchResults, words, totalCount, nil + // 按加权后的分数重新排序 + sort.Slice(searchResults, func(i, j int) bool { + return searchResults[i].Score > searchResults[j].Score + }) + + // 分页 + totalCount := len(searchResults) + offset := (pageIndex - 1) * pageSize + end := offset + pageSize + if offset > totalCount { + offset = totalCount + } + if end > totalCount { + end = totalCount + } + if offset >= end { + return nil, words, totalCount, nil + } + + return searchResults[offset:end], words, totalCount, nil } // performSearchV2 执行倒排索引搜索，返回 SearchV2Result 列表 diff --git a/models/Blog.go b/models/Blog.go index 142622cf6..d6a45b1b2 100644 --- a/models/Blog.go +++ b/models/Blog.go @@ -252,8 +252,9 @@ func (b *Blog) Save(cols ...string) error { go func(blogId int, blogTitle, blogRelease, blogContent string) { content := blogRelease if content == "" { - content = blogTitle + "\n" + blogContent + content = blogContent } + content = blogTitle + "\n" + content content = utils.StripTags(content) if err := BuildIndexForBlog(blogId, content); err != nil { logs.Error("构建Blog倒排索引失败 ->", blogId, err) diff --git a/models/ContentReverseIndex.go b/models/ContentReverseIndex.go index 0ce325a49..e77f8c563 100644 --- a/models/ContentReverseIndex.go +++ b/models/ContentReverseIndex.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math" + "sort" "github.com/beego/beego/v2/client/orm" "github.com/beego/beego/v2/core/logs" @@ -102,30 +103,22 @@ type ContentReverseIndexResult struct { WordCounts []int `json:"word_counts"` // 各个词的词频 } -// FindByWordsWithPagination 根据多个分词词汇分页批量查询结果，按IDF值排序 +// FindByWords 根据多个分词词汇查询结果，按TF-IDF值排序，返回全部匹配结果（不分页） // words: 分词词汇列表 -// pageIndex: 页码，从1开始 -// pageSize: 每页数量 -func (c *ContentReverseIndex) FindByWordsWithPagination(words []string, pageIndex, pageSize int) ([]*ContentReverseIndexResult, int, error) { +func (c *ContentReverseIndex) FindByWords(words []string) ([]*ContentReverseIndexResult, error) { if len(words) == 0 { - return nil, 0, errors.New("分词词汇列表不能为空") - } - if pageIndex <= 0 { - pageIndex = 1 - } - if pageSize <= 0 { - pageSize = 10 + return nil, errors.New("分词词汇列表不能为空") } o := orm.NewOrm() tableName := c.TableNameWithPrefix() // 计算总文档数 - totalDocsSql := "SELECT COUNT(DISTINCT CONCAT(content_type, '-', content_id)) FROM " + tableName + totalDocsSql := "SELECT COUNT(*) FROM (SELECT DISTINCT content_type, content_id FROM " + tableName + ") AS t" var totalDocs int err := o.Raw(totalDocsSql).QueryRow(&totalDocs) if err != nil { - return nil, 0, err + return nil, err } // 构建IN条件 @@ -149,102 +142,80 @@ func (c *ContentReverseIndex) FindByWordsWithPagination(words []string, pageInde var records []indexRecord _, err = o.Raw(sql, wordArgs...).QueryRows(&records) if err != nil { - return nil, 0, err - } - - // 计算各文档的总词数 - sql = "SELECT content_type, content_id, count(word_count) total_word_count FROM " + tableName + - " GROUP BY content_type, content_id" - type docWordCountRecord struct { - ContentType int - ContentId int - TotalWordCount int - } - var docWordCountRecords []docWordCountRecord - _, err = o.Raw(sql).QueryRows(&docWordCountRecords) - if err != nil { - return nil, 0, err - } - - docTotalWordCountMap := make(map[string]int) - for _, record := range docWordCountRecords { - key := fmt.Sprintf("%d-%d", record.ContentType, record.ContentId) - docTotalWordCountMap[key] = record.TotalWordCount + return nil, err } - // 聚合每个(content_type, content_id)的词频和计算TF-IDF - contentMap := make(map[string]*ContentReverseIndexResult) + // 计算每个词的文档频率（DF）：每个词出现在多少个文档中 + wordDocFreq := make(map[string]map[string]bool) for _, record := range records { key := fmt.Sprintf("%d-%d", record.ContentType, record.ContentId) - if result, exists := contentMap[key]; exists { - result.WordCounts = append(result.WordCounts, record.WordCount) - } else { - contentMap[key] = &ContentReverseIndexResult{ - ContentId: record.ContentId, - ContentType: record.ContentType, - WordCounts: []int{record.WordCount}, - } + if wordDocFreq[record.Word] == nil { + wordDocFreq[record.Word] = make(map[string]bool) } + wordDocFreq[record.Word][key] = true } - docMapWithWords := make(map[string]int) // 用于计算包含搜索词的文档数 - // 计算每个文档包含多少个查询词 - docWordCount := make(map[string]int) + // 聚合每个文档的匹配词信息 + type docWordInfo struct { + Word string + WordCount int + } + docWords := make(map[string][]docWordInfo) for _, record := range records { key := fmt.Sprintf("%d-%d", record.ContentType, record.ContentId) - docWordCount[key] += record.WordCount - docMapWithWords[key] += 1 - } - - // 计算IDF并生成结果 - results := make([]*ContentReverseIndexResult, 0, len(contentMap)) - for key := range contentMap { - result := contentMap[key] - // 计算TF：词频之和 - tf := float64(docWordCount[key]) / float64(docTotalWordCountMap[key]+1) - // 计算DF：包含该词的文档数（简化处理，使用该文档包含的查询词数量） - df := len(docMapWithWords) - // 计算IDF - idf := 0.0 - if df > 0 && totalDocs > 0 { - idf = math.Log(float64(totalDocs+1) / float64(df)) + docWords[key] = append(docWords[key], docWordInfo{ + Word: record.Word, + WordCount: record.WordCount, + }) + } + + // 计算每个文档的TF-IDF分数（使用正确的per-word IDF） + results := make([]*ContentReverseIndexResult, 0, len(docWords)) + for key, wordInfos := range docWords { + var contentType, contentId int + fmt.Sscanf(key, "%d-%d", &contentType, &contentId) + + score := 0.0 + wordCounts := make([]int, 0, len(wordInfos)) + + for _, wi := range wordInfos { + wordCounts = append(wordCounts, wi.WordCount) + // TF: 使用对数TF（sublinear TF），避免长文档被不合理惩罚 + tf := 1.0 + math.Log(float64(wi.WordCount)+1) + // IDF: 每个词独立计算，稀有词权重更高 + df := len(wordDocFreq[wi.Word]) + idf := 0.0 + if df > 0 && totalDocs > 0 { + idf = math.Log(float64(totalDocs+1) / float64(df+1)) + } + // 词长权重：长词（更具体的词）贡献更大 + wordLen := float64(len([]rune(wi.Word))) + lengthWeight := math.Log2(1.0 + wordLen) + score += tf * idf * lengthWeight } - // 用于根据文档总词数调整TF-IDF的权重，避免总词数过小的文档权重过高 - alpha := math.Log(1.0+float64(docTotalWordCountMap[key])*0.01) * 100 - // TF-IDF分数 - result.Score = float64(tf) * idf * float64(alpha) - results = append(results, result) + // 查询词覆盖率加成：匹配的查询词越多，分数越高 + coverage := float64(len(wordInfos)) / float64(len(words)) + score *= (1.0 + coverage) + + results = append(results, &ContentReverseIndexResult{ + ContentId: contentId, + ContentType: contentType, + Score: score, + WordCounts: wordCounts, + }) } // 按Score降序排序 sortResultsByScore(results) - totalCount := len(results) - // 分页 - offset := (pageIndex - 1) * pageSize - start := offset - end := offset + pageSize - if start > totalCount { - start = totalCount - } - if end > totalCount { - end = totalCount - } - if start >= end { - return nil, totalCount, nil - } - return results[start:end], totalCount, nil + return results, nil } func sortResultsByScore(results []*ContentReverseIndexResult) { - for i := 0; i < len(results)-1; i++ { - for j := i + 1; j < len(results); j++ { - if results[i].Score < results[j].Score { - results[i], results[j] = results[j], results[i] - } - } - } + sort.Slice(results, func(i, j int) bool { + return results[i].Score > results[j].Score + }) } func generateIndexId(contentType, contentId int, word string) string { @@ -446,9 +417,7 @@ func InitializeMissingDocumentIndexes() { if content == "" { content = doc.Markdown } - for i := 0; i < 10; i++ { // 标题内容"十分"重要 - content = doc.DocumentName + "\n" + content - } + content = doc.DocumentName + "\n" + content content = utils.StripTags(content) err := BuildIndexForDocument(doc.DocumentId, content) if err != nil { @@ -481,9 +450,7 @@ func InitializeMissingBlogIndexes() { if content == "" { content = blog.BlogContent } - for i := 0; i < 10; i++ { // 标题内容"十分"重要 - content = blog.BlogTitle + "\n" + content - } + content = blog.BlogTitle + "\n" + content content = utils.StripTags(content) err := BuildIndexForBlog(blog.BlogId, content) @@ -496,3 +463,105 @@ func InitializeMissingBlogIndexes() { } } } + +// RebuildAllIndexes 全量重建倒排索引（先清空再重建） +func RebuildAllIndexes() { + logs.Info("开始全量重建倒排索引...") + + // 清空倒排索引表 + o := orm.NewOrm() + tableName := NewContentReverseIndex().TableNameWithPrefix() + _, err := o.Raw("DELETE FROM " + tableName).Exec() + if err != nil { + logs.Error("清空倒排索引表失败 ->", err) + return + } + logs.Info("倒排索引表已清空") + + // 重建文档索引 + rebuildDocumentIndexes() + // 重建博客索引 + rebuildBlogIndexes() + + logs.Info("全量重建倒排索引完成") +} + +func rebuildDocumentIndexes() { + o := orm.NewOrm() + batchSize := 100 + offset := 0 + total := 0 + + for { + var documents []*Document + _, err := o.QueryTable(NewDocument().TableNameWithPrefix()). + OrderBy("document_id"). + Limit(batchSize, offset). + All(&documents) + if err != nil { + logs.Error("查询文档失败 ->", err) + break + } + if len(documents) == 0 { + break + } + + for _, doc := range documents { + content := doc.Release + if content == "" { + content = doc.Markdown + } + content = doc.DocumentName + "\n" + content + content = utils.StripTags(content) + if err := BuildIndexForDocument(doc.DocumentId, content); err != nil { + logs.Error("重建文档倒排索引失败 ->", doc.DocumentId, err) + } else { + total++ + } + } + + offset += batchSize + logs.Info("已重建文档索引:", total) + } + logs.Info("文档索引重建完成, 共:", total) +} + +func rebuildBlogIndexes() { + o := orm.NewOrm() + batchSize := 100 + offset := 0 + total := 0 + + for { + var blogs []*Blog + _, err := o.QueryTable(NewBlog().TableNameWithPrefix()). + OrderBy("blog_id"). + Limit(batchSize, offset). + All(&blogs) + if err != nil { + logs.Error("查询博客失败 ->", err) + break + } + if len(blogs) == 0 { + break + } + + for _, blog := range blogs { + content := blog.BlogRelease + if content == "" { + content = blog.BlogContent + } + content = blog.BlogTitle + "\n" + content + content = utils.StripTags(content) + if err := BuildIndexForBlog(blog.BlogId, content); err != nil { + logs.Error("重建Blog倒排索引失败 ->", blog.BlogId, err) + } else { + total++ + } + } + + offset += batchSize + logs.Info("已重建Blog索引:", total) + } + logs.Info("Blog索引重建完成, 共:", total) +} diff --git a/models/DocumentModel.go b/models/DocumentModel.go index 2533b7446..8b543185a 100644 --- a/models/DocumentModel.go +++ b/models/DocumentModel.go @@ -268,10 +268,11 @@ func (item *Document) ReleaseContent() error { // 刷新倒排索引 go func(docId int, docName, release, markdown string) { - content := docName + "\n" + release + content := release if content == "" { content = markdown } + content = docName + "\n" + content content = utils.StripTags(content) if err := BuildIndexForDocument(docId, content); err != nil { logs.Error("error: 构建文档倒排索引失败 ->", docId, err) diff --git a/models/DocumentSearchResult.go b/models/DocumentSearchResult.go index c93477b37..df070a855 100644 --- a/models/DocumentSearchResult.go +++ b/models/DocumentSearchResult.go @@ -84,7 +84,8 @@ FROM ( book.book_name, rel.member_id, mdmb.account AS author, - 'document' AS search_type + 'document' AS search_type, + CASE WHEN doc.document_name LIKE ? THEN 2 ELSE 1 END AS relevance FROM md_documents AS doc LEFT JOIN md_books AS book ON doc.book_id = book.book_id LEFT JOIN md_relationship AS rel ON book.book_id = rel.book_id AND rel.role_id = 0 @@ -102,7 +103,8 @@ SELECT book.book_name, rel.member_id, mdmb.account AS author, - 'book' AS search_type + 'book' AS search_type, + CASE WHEN book.book_name LIKE ? THEN 2 ELSE 1 END AS relevance FROM md_books AS book LEFT JOIN md_relationship AS rel ON book.book_id = rel.book_id AND rel.role_id = 0 LEFT JOIN md_members AS mdmb ON rel.member_id = mdmb.member_id @@ -120,12 +122,13 @@ WHERE book.privately_owned = 0 AND (book.book_name LIKE ? OR book.description LI blog.blog_title as book_name, blog.member_id, mdmb.account, - 'blog' AS search_type + 'blog' AS search_type, + CASE WHEN blog.blog_title LIKE ? THEN 2 ELSE 1 END AS relevance FROM md_blogs AS blog LEFT JOIN md_members AS mdmb ON blog.member_id = mdmb.member_id WHERE blog.blog_status = 'public' AND (blog.blog_release LIKE ? OR blog.blog_title LIKE ?) ) AS union_table -ORDER BY create_time DESC +ORDER BY relevance DESC, create_time DESC LIMIT ? OFFSET ?;` err = o.Raw(escape_sql(sql1), keyword, keyword).QueryRow(&totalCount) @@ -158,7 +161,7 @@ WHERE book.privately_owned = 0 AND (book.book_name LIKE ? OR book.description LI totalCount += c - _, err = o.Raw(escape_sql(sql2), keyword, keyword, keyword, keyword, keyword, keyword, pageSize, offset).QueryRows(&searchResult) + _, err = o.Raw(escape_sql(sql2), keyword, keyword, keyword, keyword, keyword, keyword, keyword, keyword, keyword, pageSize, offset).QueryRows(&searchResult) if err != nil { logs.Error("查询搜索结果失败 -> ", err) return @@ -187,7 +190,8 @@ FROM ( book.book_name, rel.member_id, mdmb.account AS author, - 'document' AS search_type + 'document' AS search_type, + CASE WHEN doc.document_name LIKE ? THEN 2 ELSE 1 END AS relevance FROM md_documents AS doc LEFT JOIN md_books AS book ON doc.book_id = book.book_id LEFT JOIN md_relationship AS rel ON book.book_id = rel.book_id AND rel.role_id = 0 @@ -218,7 +222,8 @@ FROM ( book.book_name, rel.member_id, mdmb.account AS author, - 'book' AS search_type + 'book' AS search_type, + CASE WHEN book.book_name LIKE ? THEN 2 ELSE 1 END AS relevance FROM md_books AS book LEFT JOIN md_relationship AS rel ON book.book_id = rel.book_id AND rel.role_id = 0 LEFT JOIN md_members AS mdmb ON rel.member_id = mdmb.member_id @@ -247,13 +252,14 @@ FROM ( blog.blog_title as book_name, blog.member_id, mdmb.account, - 'blog' AS search_type + 'blog' AS search_type, + CASE WHEN blog.blog_title LIKE ? THEN 2 ELSE 1 END AS relevance FROM md_blogs AS blog LEFT JOIN md_members AS mdmb ON blog.member_id = mdmb.member_id WHERE (blog.blog_status = 'public' OR blog.member_id = ?) AND blog.blog_type = 0 AND (blog.blog_release LIKE ? OR blog.blog_title LIKE ?) ) AS union_table -ORDER BY create_time DESC +ORDER BY relevance DESC, create_time DESC LIMIT ? OFFSET ?;` err = o.Raw(escape_sql(sql1), memberId, memberId, keyword, keyword).QueryRow(&totalCount) @@ -292,7 +298,7 @@ WHERE (book.privately_owned = 0 OR rel1.relationship_id > 0 or team.team_member_ totalCount += c - _, err = o.Raw(escape_sql(sql2), memberId, memberId, keyword, keyword, memberId, memberId, keyword, keyword, memberId, keyword, keyword, pageSize, offset).QueryRows(&searchResult) + _, err = o.Raw(escape_sql(sql2), keyword, memberId, memberId, keyword, keyword, keyword, memberId, memberId, keyword, keyword, keyword, memberId, keyword, keyword, pageSize, offset).QueryRows(&searchResult) if err != nil { return } diff --git a/utils/segmenter/segmenter.go b/utils/segmenter/segmenter.go index ec3b9acc3..6769be01e 100644 --- a/utils/segmenter/segmenter.go +++ b/utils/segmenter/segmenter.go @@ -5,6 +5,7 @@ import ( "path/filepath" "strings" "sync" + "unicode" "github.com/beego/beego/v2/core/logs" "github.com/mindoc-org/mindoc/conf" @@ -67,6 +68,11 @@ func Segment(text string) []string { } // 转小写（英文） word = strings.ToLower(word) + // 过滤单字符标点符号/特殊字符，避免匹配大量无关文档 + runes := []rune(word) + if len(runes) == 1 && !unicode.IsLetter(runes[0]) && !unicode.IsDigit(runes[0]) { + continue + } result = append(result, word) }