nune/gitea
nune/gitea
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Move issue pin to an standalone table for querying performance (#33452)

Browse source 
Noticed a SQL in gitea.com has a bigger load. It seems both `is_pull`
and `pin_order` are not indexed columns in the database.

```SQL
SELECT `id`, `repo_id`, `index`, `poster_id`, `original_author`, `original_author_id`, `name`, `content`, `content_version`, `milestone_id`, `priority`, `is_closed`, `is_pull`, `num_comments`, `ref`, `pin_order`, `deadline_unix`, `created_unix`, `updated_unix`, `closed_unix`, `is_locked`, `time_estimate` FROM `issue` WHERE (repo_id =?) AND (is_pull = 0) AND (pin_order > 0) ORDER BY pin_order
```

I came across a comment
https://github.com/go-gitea/gitea/pull/24406#issuecomment-1527747296
from @delvh , which presents a more reasonable approach. Based on this,
this PR will migrate all issue and pull request pin data from the
`issue` table to the `issue_pin` table. This change benefits larger
Gitea instances by improving scalability and performance.

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Lunny Xiao 2025-02-17 11:28:37 -08:00 • committed by GitHub
parent f5a81f9636
commit 7df09e31fa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 396 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

6
models/fixtures/issue_pin.yml Normal file
View file

@ -0,0 +1,6 @@
-
id: 1
repo_id: 2
issue_id: 4
is_pull: false
pin_order: 1

215
models/issues/issue.go
View file

@ -97,7 +97,7 @@ type Issue struct {
// TODO: RemoveIssueRef: see "repo/issue/branch_selector_field.tmpl"
Ref string
PinOrder int `xorm:"DEFAULT 0"`
PinOrder int `xorm:"-"` // 0 means not loaded, -1 means loaded but not pinned
DeadlineUnix timeutil.TimeStamp `xorm:"INDEX"`
@ -291,6 +291,23 @@ func (issue *Issue) LoadMilestone(ctx context.Context) (err error) {
return nil
}
func (issue *Issue) LoadPinOrder(ctx context.Context) error {
if issue.PinOrder != 0 {
return nil
}
issuePin, err := GetIssuePin(ctx, issue)
if err != nil && !db.IsErrNotExist(err) {
return err
}
if issuePin != nil {
issue.PinOrder = issuePin.PinOrder
} else {
issue.PinOrder = -1
}
return nil
}
// LoadAttributes loads the attribute of this issue.
func (issue *Issue) LoadAttributes(ctx context.Context) (err error) {
if err = issue.LoadRepo(ctx); err != nil {
@ -330,6 +347,10 @@ func (issue *Issue) LoadAttributes(ctx context.Context) (err error) {
return err
}
if err = issue.LoadPinOrder(ctx); err != nil {
return err
}
if err = issue.Comments.LoadAttributes(ctx); err != nil {
return err
}
@ -342,6 +363,14 @@ func (issue *Issue) LoadAttributes(ctx context.Context) (err error) {
return issue.loadReactions(ctx)
}
// IsPinned returns if a Issue is pinned
func (issue *Issue) IsPinned() bool {
if issue.PinOrder == 0 {
setting.PanicInDevOrTesting("issue's pinorder has not been loaded")
}
return issue.PinOrder > 0
}
func (issue *Issue) ResetAttributesLoaded() {
issue.isLabelsLoaded = false
issue.isMilestoneLoaded = false
@ -720,190 +749,6 @@ func (issue *Issue) HasOriginalAuthor() bool {
return issue.OriginalAuthor != "" && issue.OriginalAuthorID != 0
}
var ErrIssueMaxPinReached = util.NewInvalidArgumentErrorf("the max number of pinned issues has been readched")
// IsPinned returns if a Issue is pinned
func (issue *Issue) IsPinned() bool {
return issue.PinOrder != 0
}
// Pin pins a Issue
func (issue *Issue) Pin(ctx context.Context, user *user_model.User) error {
// If the Issue is already pinned, we don't need to pin it twice
if issue.IsPinned() {
return nil
}
var maxPin int
_, err := db.GetEngine(ctx).SQL("SELECT MAX(pin_order) FROM issue WHERE repo_id = ? AND is_pull = ?", issue.RepoID, issue.IsPull).Get(&maxPin)
if err != nil {
return err
}
// Check if the maximum allowed Pins reached
if maxPin >= setting.Repository.Issue.MaxPinned {
return ErrIssueMaxPinReached
}
_, err = db.GetEngine(ctx).Table("issue").
Where("id = ?", issue.ID).
Update(map[string]any{
"pin_order": maxPin + 1,
})
if err != nil {
return err
}
// Add the pin event to the history
opts := &CreateCommentOptions{
Type: CommentTypePin,
Doer: user,
Repo: issue.Repo,
Issue: issue,
}
if _, err = CreateComment(ctx, opts); err != nil {
return err
}
return nil
}
// UnpinIssue unpins a Issue
func (issue *Issue) Unpin(ctx context.Context, user *user_model.User) error {
// If the Issue is not pinned, we don't need to unpin it
if !issue.IsPinned() {
return nil
}
// This sets the Pin for all Issues that come after the unpined Issue to the correct value
_, err := db.GetEngine(ctx).Exec("UPDATE issue SET pin_order = pin_order - 1 WHERE repo_id = ? AND is_pull = ? AND pin_order > ?", issue.RepoID, issue.IsPull, issue.PinOrder)
if err != nil {
return err
}
_, err = db.GetEngine(ctx).Table("issue").
Where("id = ?", issue.ID).
Update(map[string]any{
"pin_order": 0,
})
if err != nil {
return err
}
// Add the unpin event to the history
opts := &CreateCommentOptions{
Type: CommentTypeUnpin,
Doer: user,
Repo: issue.Repo,
Issue: issue,
}
if _, err = CreateComment(ctx, opts); err != nil {
return err
}
return nil
}
// PinOrUnpin pins or unpins a Issue
func (issue *Issue) PinOrUnpin(ctx context.Context, user *user_model.User) error {
if !issue.IsPinned() {
return issue.Pin(ctx, user)
}
return issue.Unpin(ctx, user)
}
// MovePin moves a Pinned Issue to a new Position
func (issue *Issue) MovePin(ctx context.Context, newPosition int) error {
// If the Issue is not pinned, we can't move them
if !issue.IsPinned() {
return nil
}
if newPosition < 1 {
return fmt.Errorf("The Position can't be lower than 1")
}
dbctx, committer, err := db.TxContext(ctx)
if err != nil {
return err
}
defer committer.Close()
var maxPin int
_, err = db.GetEngine(dbctx).SQL("SELECT MAX(pin_order) FROM issue WHERE repo_id = ? AND is_pull = ?", issue.RepoID, issue.IsPull).Get(&maxPin)
if err != nil {
return err
}
// If the new Position bigger than the current Maximum, set it to the Maximum
if newPosition > maxPin+1 {
newPosition = maxPin + 1
}
// Lower the Position of all Pinned Issue that came after the current Position
_, err = db.GetEngine(dbctx).Exec("UPDATE issue SET pin_order = pin_order - 1 WHERE repo_id = ? AND is_pull = ? AND pin_order > ?", issue.RepoID, issue.IsPull, issue.PinOrder)
if err != nil {
return err
}
// Higher the Position of all Pinned Issues that comes after the new Position
_, err = db.GetEngine(dbctx).Exec("UPDATE issue SET pin_order = pin_order + 1 WHERE repo_id = ? AND is_pull = ? AND pin_order >= ?", issue.RepoID, issue.IsPull, newPosition)
if err != nil {
return err
}
_, err = db.GetEngine(dbctx).Table("issue").
Where("id = ?", issue.ID).
Update(map[string]any{
"pin_order": newPosition,
})
if err != nil {
return err
}
return committer.Commit()
}
// GetPinnedIssues returns the pinned Issues for the given Repo and type
func GetPinnedIssues(ctx context.Context, repoID int64, isPull bool) (IssueList, error) {
issues := make(IssueList, 0)
err := db.GetEngine(ctx).
Table("issue").
Where("repo_id = ?", repoID).
And("is_pull = ?", isPull).
And("pin_order > 0").
OrderBy("pin_order").
Find(&issues)
if err != nil {
return nil, err
}
err = issues.LoadAttributes(ctx)
if err != nil {
return nil, err
}
return issues, nil
}
// IsNewPinAllowed returns if a new Issue or Pull request can be pinned
func IsNewPinAllowed(ctx context.Context, repoID int64, isPull bool) (bool, error) {
var maxPin int
_, err := db.GetEngine(ctx).SQL("SELECT COUNT(pin_order) FROM issue WHERE repo_id = ? AND is_pull = ? AND pin_order > 0", repoID, isPull).Get(&maxPin)
if err != nil {
return false, err
}
return maxPin < setting.Repository.Issue.MaxPinned, nil
}
// IsErrIssueMaxPinReached returns if the error is, that the User can't pin more Issues
func IsErrIssueMaxPinReached(err error) bool {
return err == ErrIssueMaxPinReached
}
// InsertIssues insert issues to database
func InsertIssues(ctx context.Context, issues ...*Issue) error {
ctx, committer, err := db.TxContext(ctx)

33
models/issues/issue_list.go
View file

@ -506,6 +506,39 @@ func (issues IssueList) loadTotalTrackedTimes(ctx context.Context) (err error) {
return nil
}
func (issues IssueList) LoadPinOrder(ctx context.Context) error {
if len(issues) == 0 {
return nil
}
issueIDs := container.FilterSlice(issues, func(issue *Issue) (int64, bool) {
return issue.ID, issue.PinOrder == 0
})
if len(issueIDs) == 0 {
return nil
}
issuePins, err := GetIssuePinsByIssueIDs(ctx, issueIDs)
if err != nil {
return err
}
for _, issue := range issues {
if issue.PinOrder != 0 {
continue
}
for _, pin := range issuePins {
if pin.IssueID == issue.ID {
issue.PinOrder = pin.PinOrder
break
}
}
if issue.PinOrder == 0 {
issue.PinOrder = -1
}
}
return nil
}
// loadAttributes loads all attributes, expect for attachments and comments
func (issues IssueList) LoadAttributes(ctx context.Context) error {
if _, err := issues.LoadRepositories(ctx); err != nil {

246
models/issues/issue_pin.go Normal file
View file

@ -0,0 +1,246 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package issues
import (
"context"
"errors"
"sort"
"code.gitea.io/gitea/models/db"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
type IssuePin struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(s) NOT NULL"`
IssueID int64 `xorm:"UNIQUE(s) NOT NULL"`
IsPull bool `xorm:"NOT NULL"`
PinOrder int `xorm:"DEFAULT 0"`
}
var ErrIssueMaxPinReached = util.NewInvalidArgumentErrorf("the max number of pinned issues has been readched")
// IsErrIssueMaxPinReached returns if the error is, that the User can't pin more Issues
func IsErrIssueMaxPinReached(err error) bool {
return err == ErrIssueMaxPinReached
}
func init() {
db.RegisterModel(new(IssuePin))
}
func GetIssuePin(ctx context.Context, issue *Issue) (*IssuePin, error) {
pin := new(IssuePin)
has, err := db.GetEngine(ctx).
Where("repo_id = ?", issue.RepoID).
And("issue_id = ?", issue.ID).Get(pin)
if err != nil {
return nil, err
} else if !has {
return nil, db.ErrNotExist{
Resource: "IssuePin",
ID: issue.ID,
}
}
return pin, nil
}
func GetIssuePinsByIssueIDs(ctx context.Context, issueIDs []int64) ([]IssuePin, error) {
var pins []IssuePin
if err := db.GetEngine(ctx).In("issue_id", issueIDs).Find(&pins); err != nil {
return nil, err
}
return pins, nil
}
// Pin pins a Issue
func PinIssue(ctx context.Context, issue *Issue, user *user_model.User) error {
return db.WithTx(ctx, func(ctx context.Context) error {
pinnedIssuesNum, err := getPinnedIssuesNum(ctx, issue.RepoID, issue.IsPull)
if err != nil {
return err
}
// Check if the maximum allowed Pins reached
if pinnedIssuesNum >= setting.Repository.Issue.MaxPinned {
return ErrIssueMaxPinReached
}
pinnedIssuesMaxPinOrder, err := getPinnedIssuesMaxPinOrder(ctx, issue.RepoID, issue.IsPull)
if err != nil {
return err
}
if _, err = db.GetEngine(ctx).Insert(&IssuePin{
RepoID: issue.RepoID,
IssueID: issue.ID,
IsPull: issue.IsPull,
PinOrder: pinnedIssuesMaxPinOrder + 1,
}); err != nil {
return err
}
// Add the pin event to the history
_, err = CreateComment(ctx, &CreateCommentOptions{
Type: CommentTypePin,
Doer: user,
Repo: issue.Repo,
Issue: issue,
})
return err
})
}
// UnpinIssue unpins a Issue
func UnpinIssue(ctx context.Context, issue *Issue, user *user_model.User) error {
return db.WithTx(ctx, func(ctx context.Context) error {
// This sets the Pin for all Issues that come after the unpined Issue to the correct value
cnt, err := db.GetEngine(ctx).Where("issue_id=?", issue.ID).Delete(new(IssuePin))
if err != nil {
return err
}
if cnt == 0 {
return nil
}
// Add the unpin event to the history
_, err = CreateComment(ctx, &CreateCommentOptions{
Type: CommentTypeUnpin,
Doer: user,
Repo: issue.Repo,
Issue: issue,
})
return err
})
}
func getPinnedIssuesNum(ctx context.Context, repoID int64, isPull bool) (int, error) {
var pinnedIssuesNum int
_, err := db.GetEngine(ctx).SQL("SELECT count(pin_order) FROM issue_pin WHERE repo_id = ? AND is_pull = ?", repoID, isPull).Get(&pinnedIssuesNum)
return pinnedIssuesNum, err
}
func getPinnedIssuesMaxPinOrder(ctx context.Context, repoID int64, isPull bool) (int, error) {
var maxPinnedIssuesMaxPinOrder int
_, err := db.GetEngine(ctx).SQL("SELECT max(pin_order) FROM issue_pin WHERE repo_id = ? AND is_pull = ?", repoID, isPull).Get(&maxPinnedIssuesMaxPinOrder)
return maxPinnedIssuesMaxPinOrder, err
}
// MovePin moves a Pinned Issue to a new Position
func MovePin(ctx context.Context, issue *Issue, newPosition int) error {
if newPosition < 1 {
return errors.New("The Position can't be lower than 1")
}
issuePin, err := GetIssuePin(ctx, issue)
if err != nil {
return err
}
if issuePin.PinOrder == newPosition {
return nil
}
return db.WithTx(ctx, func(ctx context.Context) error {
if issuePin.PinOrder > newPosition { // move the issue to a lower position
_, err = db.GetEngine(ctx).Exec("UPDATE issue_pin SET pin_order = pin_order + 1 WHERE repo_id = ? AND is_pull = ? AND pin_order >= ? AND pin_order < ?", issue.RepoID, issue.IsPull, newPosition, issuePin.PinOrder)
} else { // move the issue to a higher position
// Lower the Position of all Pinned Issue that came after the current Position
_, err = db.GetEngine(ctx).Exec("UPDATE issue_pin SET pin_order = pin_order - 1 WHERE repo_id = ? AND is_pull = ? AND pin_order > ? AND pin_order <= ?", issue.RepoID, issue.IsPull, issuePin.PinOrder, newPosition)
}
if err != nil {
return err
}
_, err = db.GetEngine(ctx).
Table("issue_pin").
Where("id = ?", issuePin.ID).
Update(map[string]any{
"pin_order": newPosition,
})
return err
})
}
func GetPinnedIssueIDs(ctx context.Context, repoID int64, isPull bool) ([]int64, error) {
var issuePins []IssuePin
if err := db.GetEngine(ctx).
Table("issue_pin").
Where("repo_id = ?", repoID).
And("is_pull = ?", isPull).
Find(&issuePins); err != nil {
return nil, err
}
sort.Slice(issuePins, func(i, j int) bool {
return issuePins[i].PinOrder < issuePins[j].PinOrder
})
var ids []int64
for _, pin := range issuePins {
ids = append(ids, pin.IssueID)
}
return ids, nil
}
func GetIssuePinsByRepoID(ctx context.Context, repoID int64, isPull bool) ([]*IssuePin, error) {
var pins []*IssuePin
if err := db.GetEngine(ctx).Where("repo_id = ? AND is_pull = ?", repoID, isPull).Find(&pins); err != nil {
return nil, err
}
return pins, nil
}
// GetPinnedIssues returns the pinned Issues for the given Repo and type
func GetPinnedIssues(ctx context.Context, repoID int64, isPull bool) (IssueList, error) {
issuePins, err := GetIssuePinsByRepoID(ctx, repoID, isPull)
if err != nil {
return nil, err
}
if len(issuePins) == 0 {
return IssueList{}, nil
}
ids := make([]int64, 0, len(issuePins))
for _, pin := range issuePins {
ids = append(ids, pin.IssueID)
}
issues := make(IssueList, 0, len(ids))
if err := db.GetEngine(ctx).In("id", ids).Find(&issues); err != nil {
return nil, err
}
for _, issue := range issues {
for _, pin := range issuePins {
if pin.IssueID == issue.ID {
issue.PinOrder = pin.PinOrder
break
}
}
if (!setting.IsProd || setting.IsInTesting) && issue.PinOrder == 0 {
panic("It should not happen that a pinned Issue has no PinOrder")
}
}
sort.Slice(issues, func(i, j int) bool {
return issues[i].PinOrder < issues[j].PinOrder
})
if err = issues.LoadAttributes(ctx); err != nil {
return nil, err
}
return issues, nil
}
// IsNewPinAllowed returns if a new Issue or Pull request can be pinned
func IsNewPinAllowed(ctx context.Context, repoID int64, isPull bool) (bool, error) {
var maxPin int
_, err := db.GetEngine(ctx).SQL("SELECT COUNT(pin_order) FROM issue_pin WHERE repo_id = ? AND is_pull = ?", repoID, isPull).Get(&maxPin)
if err != nil {
return false, err
}
return maxPin < setting.Repository.Issue.MaxPinned, nil
}

1
models/migrations/migrations.go
View file

@ -373,6 +373,7 @@ func prepareMigrationTasks() []*migration {
// Gitea 1.23.0-rc0 ends at migration ID number 311 (database version 312)
newMigration(312, "Add DeleteBranchAfterMerge to AutoMerge", v1_24.AddDeleteBranchAfterMergeForAutoMerge),
newMigration(313, "Move PinOrder from issue table to a new table issue_pin", v1_24.MovePinOrderToTableIssuePin),
}
return preparedMigrations
}

31
models/migrations/v1_24/v313.go Normal file
View file

@ -0,0 +1,31 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_24 //nolint
import (
"code.gitea.io/gitea/models/migrations/base"
"xorm.io/xorm"
)
func MovePinOrderToTableIssuePin(x *xorm.Engine) error {
type IssuePin struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(s) NOT NULL"`
IssueID int64 `xorm:"UNIQUE(s) NOT NULL"`
IsPull bool `xorm:"NOT NULL"`
PinOrder int `xorm:"DEFAULT 0"`
}
if err := x.Sync(new(IssuePin)); err != nil {
return err
}
if _, err := x.Exec("INSERT INTO issue_pin (repo_id, issue_id, is_pull, pin_order) SELECT repo_id, id, is_pull, pin_order FROM issue WHERE pin_order > 0"); err != nil {
return err
}
sess := x.NewSession()
defer sess.Close()
return base.DropTableColumns(sess, "issue", "pin_order")
}