Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add search by index to issues api #2606

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions models/issue_indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package models

import (
"fmt"
"strconv"

"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/log"
Expand Down Expand Up @@ -88,10 +89,11 @@ func (issue *Issue) update() indexer.IssueIndexerUpdate {
return indexer.IssueIndexerUpdate{
IssueID: issue.ID,
Data: &indexer.IssueIndexerData{
RepoID: issue.RepoID,
Title: issue.Title,
Content: issue.Content,
Comments: comments,
RepoID: issue.RepoID,
Title: issue.Title,
Content: issue.Content,
Comments: comments,
IssueIndex: strconv.FormatInt(issue.Index, 10),
},
}
}
Expand Down
2 changes: 2 additions & 0 deletions models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,8 @@ var migrations = []Migration{
NewMigration("add tags to releases and sync existing repositories", releaseAddColumnIsTagAndSyncTags),
// v43 -> v44
NewMigration("fix protected branch can push value to false", fixProtectedBranchCanPushValue),
// v44 -> v45
NewMigration("add index field to bleve issue index", regenerateIssueIndexer),
}

// Migrate database to current version
Expand Down
24 changes: 24 additions & 0 deletions models/migrations/v44.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
"os"

"code.gitea.io/gitea/modules/setting"
"github.com/go-xorm/xorm"
)

func regenerateIssueIndexer(x *xorm.Engine) error {
_, err := os.Stat(setting.Indexer.IssuePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}

return os.RemoveAll(setting.Indexer.IssuePath)
}
6 changes: 6 additions & 0 deletions modules/indexer/indexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ func newMatchPhraseQuery(matchPhrase, field, analyzer string) *query.MatchPhrase
return q
}

func newPrefixQuery(matchPhrase, field string) *query.PrefixQuery {
q := bleve.NewPrefixQuery(matchPhrase)
q.FieldVal = field
return q
}

const unicodeNormalizeName = "unicodeNormalize"

func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
Expand Down
51 changes: 35 additions & 16 deletions modules/indexer/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package indexer

import (
"os"
"strconv"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
Expand All @@ -22,10 +23,11 @@ var issueIndexer bleve.Index

// IssueIndexerData data stored in the issue indexer
type IssueIndexerData struct {
RepoID int64
Title string
Content string
Comments []string
RepoID int64
Title string
Content string
Comments []string
IssueIndex string
}

// IssueIndexerUpdate an update to the issue indexer
Expand Down Expand Up @@ -76,6 +78,7 @@ func createIssueIndexer() error {
docMapping.AddFieldMappingsAt("Title", textFieldMapping)
docMapping.AddFieldMappingsAt("Content", textFieldMapping)
docMapping.AddFieldMappingsAt("Comments", textFieldMapping)
docMapping.AddFieldMappingsAt("IssueIndex", textFieldMapping)

if err := addUnicodeNormalizeTokenFilter(mapping); err != nil {
return err
Expand Down Expand Up @@ -104,6 +107,23 @@ func IssueIndexerBatch() *Batch {
}
}

func searchIssues(req *bleve.SearchRequest) ([]int64, error) {
result, err := issueIndexer.Search(req)
if err != nil {
return nil, err
}

issueIDs := make([]int64, len(result.Hits))
for i, hit := range result.Hits {
issueIDs[i], err = idOfIndexerID(hit.ID)
if err != nil {
return nil, err
}
}

return issueIDs, nil
}

// SearchIssuesByKeyword searches for issues by given conditions.
// Returns the matching issue IDs
func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
Expand All @@ -116,17 +136,16 @@ func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
))
search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)

result, err := issueIndexer.Search(search)
if err != nil {
return nil, err
}
return searchIssues(search)
}

issueIDs := make([]int64, len(result.Hits))
for i, hit := range result.Hits {
issueIDs[i], err = idOfIndexerID(hit.ID)
if err != nil {
return nil, err
}
}
return issueIDs, nil
// SearchIssuesByIndex searches for issues by given conditions.
// Returns the matching issue IDs
func SearchIssuesByIndex(repoID, issueIndex int64) ([]int64, error) {
Copy link
Member

@ethantkoenig ethantkoenig Sep 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure using the indexer here is the best approach.

First of all, existing indexer entries won't have an IssueIndex field, so this feature won't work without some sort of indexer migration (which right now we don't support)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you know good and effective solution for searching numbers like this?

indexerQuery := bleve.NewConjunctionQuery(
numericEqualityQuery(repoID, "RepoID"),
newPrefixQuery(strconv.FormatInt(issueIndex, 10), "IssueIndex"))
search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)

return searchIssues(search)
}
48 changes: 35 additions & 13 deletions routers/api/v1/repo/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/indexer"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
Expand All @@ -28,21 +29,42 @@ func ListIssues(ctx *context.APIContext) {
isClosed = util.OptionalBoolFalse
}

issues, err := models.Issues(&models.IssuesOptions{
RepoID: ctx.Repo.Repository.ID,
Page: ctx.QueryInt("page"),
PageSize: setting.UI.IssuePagingNum,
IsClosed: isClosed,
})
if err != nil {
ctx.Error(500, "Issues", err)
return
issueIndex := ctx.QueryInt64("index")
var forceEmpty bool
var issueIDs []int64
var err error
if issueIndex > 0 {
issueIDs, err = indexer.SearchIssuesByIndex(ctx.Repo.Repository.ID, issueIndex)
if err != nil {
ctx.Error(500, "SearchIssuesByIDPartialy", err)
return
}
if len(issueIDs) == 0 {
forceEmpty = true
}
}

err = models.IssueList(issues).LoadAttributes()
if err != nil {
ctx.Error(500, "LoadAttributes", err)
return
var issues []*models.Issue
if forceEmpty {
issues = []*models.Issue{}
} else {
issues, err = models.Issues(&models.IssuesOptions{
RepoID: ctx.Repo.Repository.ID,
Page: ctx.QueryInt("page"),
PageSize: setting.UI.IssuePagingNum,
IsClosed: isClosed,
IssueIDs: issueIDs,
})
if err != nil {
ctx.Error(500, "Issues", err)
return
}

err = models.IssueList(issues).LoadAttributes()
if err != nil {
ctx.Error(500, "LoadAttributes", err)
return
}
}

apiIssues := make([]*api.Issue, len(issues))
Expand Down