nune/gitea
nune/gitea
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Move migrating repository from frontend to backend (#6200)

Browse source 
* move migrating to backend

* add loading image when migrating and fix tests

* fix format

* fix lint

* add redis task queue support and improve docs

* add redis vendor

* fix vet

* add database migrations and fix app.ini sample

* add comments for task section on app.ini.sample

* Update models/migrations/v84.go

Co-Authored-By: lunny <xiaolunwen@gmail.com>

* Update models/repo.go

Co-Authored-By: lunny <xiaolunwen@gmail.com>

* move migrating to backend

* add loading image when migrating and fix tests

* fix fmt

* add redis task queue support and improve docs

* fix fixtures

* fix fixtures

* fix duplicate function on index.js

* fix tests

* rename repository statuses

* check if repository is being create when SSH request

* fix lint

* fix template

* some improvements

* fix template

* unified migrate options

* fix lint

* fix loading page

* refactor

* When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration

* fix js

* Update models/repo.go

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Update docs/content/doc/advanced/config-cheat-sheet.en-us.md

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* fix tests

* rename ErrTaskIsNotExist to ErrTaskDoesNotExist

* delete release after add one on tests to make it run happy

* fix tests

* fix tests

* improve codes

* fix lint

* fix lint

* fix migrations
This commit is contained in:
Lunny Xiao 2019-10-13 21:23:14 +08:00 • committed by GitHub
parent 0a96e59884
commit f2a3abc683
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 1192 additions and 222 deletions
Download patch file Download diff file Expand all files Collapse all files

85
models/repo.go
View file

@ -126,6 +126,15 @@ func NewRepoContext() {
RemoveAllWithNotice("Clean up repository temporary data", filepath.Join(setting.AppDataPath, "tmp"))
}
// RepositoryStatus defines the status of repository
type RepositoryStatus int
// all kinds of RepositoryStatus
const (
RepositoryReady RepositoryStatus = iota // a normal repository
RepositoryBeingMigrated // repository is migrating
)
// Repository represents a git repository.
type Repository struct {
ID int64 `xorm:"pk autoincr"`
@ -156,9 +165,9 @@ type Repository struct {
IsPrivate bool `xorm:"INDEX"`
IsEmpty bool `xorm:"INDEX"`
IsArchived bool `xorm:"INDEX"`
IsMirror bool `xorm:"INDEX"`
*Mirror `xorm:"-"`
IsMirror bool `xorm:"INDEX"`
*Mirror `xorm:"-"`
Status RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
ExternalMetas map[string]string `xorm:"-"`
Units []*RepoUnit `xorm:"-"`
@ -197,6 +206,16 @@ func (repo *Repository) ColorFormat(s fmt.State) {
repo.Name)
}
// IsBeingMigrated indicates that repository is being migtated
func (repo *Repository) IsBeingMigrated() bool {
return repo.Status == RepositoryBeingMigrated
}
// IsBeingCreated indicates that repository is being migrated or forked
func (repo *Repository) IsBeingCreated() bool {
return repo.IsBeingMigrated()
}
// AfterLoad is invoked from XORM after setting the values of all fields of this object.
func (repo *Repository) AfterLoad() {
// FIXME: use models migration to solve all at once.
@ -884,18 +903,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) {
return repo.cloneLink(x, false)
}
// MigrateRepoOptions contains the repository migrate options
type MigrateRepoOptions struct {
Name string
Description string
OriginalURL string
IsPrivate bool
IsMirror bool
RemoteAddr string
Wiki bool // include wiki repository
SyncReleasesWithTags bool // sync releases from tags
}
/*
GitHub, GitLab, Gogs: *.wiki.git
BitBucket: *.git/wiki
@ -915,20 +922,28 @@ func wikiRemoteURL(remote string) string {
return ""
}
// MigrateRepository migrates an existing repository from other project hosting.
func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, error) {
repo, err := CreateRepository(doer, u, CreateRepoOptions{
Name: opts.Name,
Description: opts.Description,
OriginalURL: opts.OriginalURL,
IsPrivate: opts.IsPrivate,
IsMirror: opts.IsMirror,
})
if err != nil {
return nil, err
// CheckCreateRepository check if could created a repository
func CheckCreateRepository(doer, u *User, name string) error {
if !doer.CanCreateRepo() {
return ErrReachLimitOfRepo{u.MaxRepoCreation}
}
repoPath := RepoPath(u.Name, opts.Name)
if err := IsUsableRepoName(name); err != nil {
return err
}
has, err := isRepositoryExist(x, u, name)
if err != nil {
return fmt.Errorf("IsRepositoryExist: %v", err)
} else if has {
return ErrRepoAlreadyExist{u.Name, name}
}
return nil
}
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) {
repoPath := RepoPath(u.Name, opts.RepoName)
if u.IsOrganization() {
t, err := u.GetOwnerTeam()
@ -942,11 +957,12 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
if err := os.RemoveAll(repoPath); err != nil {
var err error
if err = os.RemoveAll(repoPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
}
if err = git.Clone(opts.RemoteAddr, repoPath, git.CloneRepoOptions{
if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
@ -955,8 +971,8 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
}
if opts.Wiki {
wikiPath := WikiPath(u.Name, opts.Name)
wikiRemotePath := wikiRemoteURL(opts.RemoteAddr)
wikiPath := WikiPath(u.Name, opts.RepoName)
wikiRemotePath := wikiRemoteURL(opts.CloneAddr)
if len(wikiRemotePath) > 0 {
if err := os.RemoveAll(wikiPath); err != nil {
return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
@ -986,7 +1002,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
return repo, fmt.Errorf("git.IsEmpty: %v", err)
}
if opts.SyncReleasesWithTags && !repo.IsEmpty {
if !opts.Releases && !repo.IsEmpty {
// Try to get HEAD branch and set it as default branch.
headBranch, err := gitRepo.GetHEADBranch()
if err != nil {
@ -1005,7 +1021,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
log.Error("Failed to update size for repository: %v", err)
}
if opts.IsMirror {
if opts.Mirror {
if _, err = x.InsertOne(&Mirror{
RepoID: repo.ID,
Interval: setting.Mirror.DefaultInterval,
@ -1143,6 +1159,7 @@ type CreateRepoOptions struct {
IsPrivate bool
IsMirror bool
AutoInit bool
Status RepositoryStatus
}
func getRepoInitFile(tp, name string) ([]byte, error) {
@ -1410,6 +1427,7 @@ func CreateRepository(doer, u *User, opts CreateRepoOptions) (_ *Repository, err
IsPrivate: opts.IsPrivate,
IsFsckEnabled: !opts.IsMirror,
CloseIssuesViaCommitInAnyBranch: setting.Repository.DefaultCloseIssuesViaCommitsInAnyBranch,
Status: opts.Status,
}
sess := x.NewSession()
@ -1856,6 +1874,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
&CommitStatus{RepoID: repoID},
&RepoIndexerStatus{RepoID: repoID},
&Comment{RefRepoID: repoID},
&Task{RepoID: repoID},
); err != nil {
return fmt.Errorf("deleteBeans: %v", err)
}