Template Repositories (#8768)

* Start work on templates Signed-off-by: jolheiser <john.olheiser@gmail.com> * Continue work Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix IsTemplate vs IsGenerated Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix tabs vs spaces * Tabs vs Spaces * Add templates to API & start adding tests Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix integration tests Signed-off-by: jolheiser <john.olheiser@gmail.com> * Remove unused User Signed-off-by: jolheiser <john.olheiser@gmail.com> * Move template tests to existing repos Signed-off-by: jolheiser <john.olheiser@gmail.com> * Minor re-check updates and cleanup Signed-off-by: jolheiser <john.olheiser@gmail.com> * make fmt Signed-off-by: jolheiser <john.olheiser@gmail.com> * Test cleanup Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix optionalbool Signed-off-by: jolheiser <john.olheiser@gmail.com> * make fmt Signed-off-by: jolheiser <john.olheiser@gmail.com> * Test fixes and icon change Signed-off-by: jolheiser <john.olheiser@gmail.com> * Add new user and repo for tests Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix tests (finally) Signed-off-by: jolheiser <john.olheiser@gmail.com> * Update meta repo with env variables Signed-off-by: jolheiser <john.olheiser@gmail.com> * Move generation to create page Combine with repo create template Modify API search to prioritize owner for repo Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix tests and coverage Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix swagger and JS lint Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix API searching for own private repos Signed-off-by: jolheiser <john.olheiser@gmail.com> * Change wording Signed-off-by: jolheiser <john.olheiser@gmail.com> * Fix repo search test. User had a private repo that didn't show up Signed-off-by: jolheiser <john.olheiser@gmail.com> * Another search test fix Signed-off-by: jolheiser <john.olheiser@gmail.com> * Clarify git content Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com> * Feedback updates Signed-off-by: jolheiser <john.olheiser@gmail.com> * Add topics WIP Signed-off-by: jolheiser <john.olheiser@gmail.com> * Finish adding topics Signed-off-by: jolheiser <john.olheiser@gmail.com> * Update locale Signed-off-by: jolheiser <john.olheiser@gmail.com>
2019-11-11 09:15:29 -06:00 · 2019-11-11 09:15:29 -06:00 · 74a6add4d9
commit 74a6add4d9
parent 74bb292fe3
58 changed files with 1441 additions and 119 deletions
--- a/models/repo.go
+++ b/models/repo.go
@ -179,6 +179,9 @@ type Repository struct {
 	IsFork                          bool               `xorm:"INDEX NOT NULL DEFAULT false"`
 	ForkID                          int64              `xorm:"INDEX"`
 	BaseRepo                        *Repository        `xorm:"-"`
+	IsTemplate                      bool               `xorm:"INDEX NOT NULL DEFAULT false"`
+	TemplateID                      int64              `xorm:"INDEX"`
+	TemplateRepo                    *Repository        `xorm:"-"`
 	Size                            int64              `xorm:"NOT NULL DEFAULT 0"`
 	IndexerStatus                   *RepoIndexerStatus `xorm:"-"`
 	IsFsckEnabled                   bool               `xorm:"NOT NULL DEFAULT true"`
@ -351,6 +354,7 @@ func (repo *Repository) innerAPIFormat(e Engine, mode AccessMode, isParent bool)
 		FullName:                  repo.FullName(),
 		Description:               repo.Description,
 		Private:                   repo.IsPrivate,
+		Template:                  repo.IsTemplate,
 		Empty:                     repo.IsEmpty,
 		Archived:                  repo.IsArchived,
 		Size:                      int(repo.Size / 1024),
@ -663,6 +667,27 @@ func (repo *Repository) getBaseRepo(e Engine) (err error) {
 	return err
 }

+// IsGenerated returns whether _this_ repository was generated from a template
+func (repo *Repository) IsGenerated() bool {
+	return repo.TemplateID != 0
+}
+
+// GetTemplateRepo populates repo.TemplateRepo for a generated repository and
+// returns an error on failure (NOTE: no error is returned for
+// non-generated repositories, and TemplateRepo will be left untouched)
+func (repo *Repository) GetTemplateRepo() (err error) {
+	return repo.getTemplateRepo(x)
+}
+
+func (repo *Repository) getTemplateRepo(e Engine) (err error) {
+	if !repo.IsGenerated() {
+		return nil
+	}
+
+	repo.TemplateRepo, err = getRepositoryByID(e, repo.TemplateID)
+	return err
+}
+
 func (repo *Repository) repoPath(e Engine) string {
 	return RepoPath(repo.mustOwnerName(e), repo.Name)
 }
@ -1220,6 +1245,20 @@ type CreateRepoOptions struct {
 	Status      RepositoryStatus
 }

+// GenerateRepoOptions contains the template units to generate
+type GenerateRepoOptions struct {
+	Name        string
+	Description string
+	Private     bool
+	GitContent  bool
+	Topics      bool
+}
+
+// IsValid checks whether at least one option is chosen for generation
+func (gro GenerateRepoOptions) IsValid() bool {
+	return gro.GitContent || gro.Topics // or other items as they are added
+}
+
 func getRepoInitFile(tp, name string) ([]byte, error) {
 	cleanedName := strings.TrimLeft(path.Clean("/"+name), "/")
 	relPath := path.Join("options", tp, cleanedName)
@ -1323,8 +1362,55 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts
 	return nil
 }

-// InitRepository initializes README and .gitignore if needed.
-func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts CreateRepoOptions) (err error) {
+func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error {
+	commitTimeStr := time.Now().Format(time.RFC3339)
+	authorSig := repo.Owner.NewGitSig()
+
+	// Because this may call hooks we should pass in the environment
+	env := append(os.Environ(),
+		"GIT_AUTHOR_NAME="+authorSig.Name,
+		"GIT_AUTHOR_EMAIL="+authorSig.Email,
+		"GIT_AUTHOR_DATE="+commitTimeStr,
+		"GIT_COMMITTER_NAME="+authorSig.Name,
+		"GIT_COMMITTER_EMAIL="+authorSig.Email,
+		"GIT_COMMITTER_DATE="+commitTimeStr,
+	)
+
+	// Clone to temporary path and do the init commit.
+	templateRepoPath := templateRepo.repoPath(e)
+	_, stderr, err := process.GetManager().ExecDirEnv(
+		-1, "",
+		fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath),
+		env,
+		git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir,
+	)
+	if err != nil {
+		return fmt.Errorf("git clone: %v - %s", err, stderr)
+	}
+
+	if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
+		return fmt.Errorf("remove git dir: %v", err)
+	}
+
+	if err := git.InitRepository(tmpDir, false); err != nil {
+		return err
+	}
+
+	repoPath := repo.repoPath(e)
+	_, stderr, err = process.GetManager().ExecDirEnv(
+		-1, tmpDir,
+		fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
+		env,
+		git.GitExecutable, "remote", "add", "origin", repoPath,
+	)
+	if err != nil {
+		return fmt.Errorf("git remote add: %v - %s", err, stderr)
+	}
+
+	return initRepoCommit(tmpDir, repo.Owner)
+}
+
+func checkInitRepository(repoPath string) (err error) {
 	// Somehow the directory could exist.
 	if com.IsExist(repoPath) {
 		return fmt.Errorf("initRepository: path already exists: %s", repoPath)
@ -1336,6 +1422,14 @@ func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts C
 	} else if err = createDelegateHooks(repoPath); err != nil {
 		return fmt.Errorf("createDelegateHooks: %v", err)
 	}
+	return nil
+}
+
+// InitRepository initializes README and .gitignore if needed.
+func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts CreateRepoOptions) (err error) {
+	if err = checkInitRepository(repoPath); err != nil {
+		return err
+	}

 	tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond()))

@ -1376,6 +1470,37 @@ func initRepository(e Engine, repoPath string, u *User, repo *Repository, opts C
 	return nil
 }

+// generateRepository initializes repository from template
+func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
+	tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond()))
+
+	if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
+		return fmt.Errorf("Failed to create dir %s: %v", tmpDir, err)
+	}
+
+	defer func() {
+		if err := os.RemoveAll(tmpDir); err != nil {
+			log.Error("RemoveAll: %v", err)
+		}
+	}()
+
+	if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil {
+		return fmt.Errorf("generateRepoCommit: %v", err)
+	}
+
+	// re-fetch repo
+	if repo, err = getRepositoryByID(e, repo.ID); err != nil {
+		return fmt.Errorf("getRepositoryByID: %v", err)
+	}
+
+	repo.DefaultBranch = "master"
+	if err = updateRepository(e, repo, false); err != nil {
+		return fmt.Errorf("updateRepository: %v", err)
+	}
+
+	return nil
+}
+
 var (
 	reservedRepoNames    = []string{".", ".."}
 	reservedRepoPatterns = []string{"*.git", "*.wiki"}
@ -2523,6 +2648,28 @@ func HasForkedRepo(ownerID, repoID int64) (*Repository, bool) {
 	return repo, has
 }

+// CopyLFS copies LFS data from one repo to another
+func CopyLFS(newRepo, oldRepo *Repository) error {
+	return copyLFS(x, newRepo, oldRepo)
+}
+
+func copyLFS(e Engine, newRepo, oldRepo *Repository) error {
+	var lfsObjects []*LFSMetaObject
+	if err := e.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil {
+		return err
+	}
+
+	for _, v := range lfsObjects {
+		v.ID = 0
+		v.RepositoryID = newRepo.ID
+		if _, err := e.Insert(v); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // ForkRepository forks a repository
 func ForkRepository(doer, owner *User, oldRepo *Repository, name, desc string) (_ *Repository, err error) {
 	forkedRepo, err := oldRepo.GetUserFork(owner.ID)
@ -2593,27 +2740,73 @@ func ForkRepository(doer, owner *User, oldRepo *Repository, name, desc string) (
 		log.Error("Failed to update size for repository: %v", err)
 	}

-	// Copy LFS meta objects in new session
-	sess2 := x.NewSession()
-	defer sess2.Close()
-	if err = sess2.Begin(); err != nil {
+	return repo, CopyLFS(repo, oldRepo)
+}
+
+// GenerateRepository generates a repository from a template
+func GenerateRepository(doer, owner *User, templateRepo *Repository, opts GenerateRepoOptions) (_ *Repository, err error) {
+	repo := &Repository{
+		OwnerID:       owner.ID,
+		Owner:         owner,
+		Name:          opts.Name,
+		LowerName:     strings.ToLower(opts.Name),
+		Description:   opts.Description,
+		IsPrivate:     opts.Private,
+		IsEmpty:       !opts.GitContent || templateRepo.IsEmpty,
+		IsFsckEnabled: templateRepo.IsFsckEnabled,
+		TemplateID:    templateRepo.ID,
+	}
+
+	createSess := x.NewSession()
+	defer createSess.Close()
+	if err = createSess.Begin(); err != nil {
+		return nil, err
+	}
+
+	if err = createRepository(createSess, doer, owner, repo); err != nil {
+		return nil, err
+	}
+
+	//Commit repo to get created repo ID
+	err = createSess.Commit()
+	if err != nil {
+		return nil, err
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err = sess.Begin(); err != nil {
 		return repo, err
 	}

-	var lfsObjects []*LFSMetaObject
-	if err = sess2.Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil {
+	repoPath := RepoPath(owner.Name, repo.Name)
+	if err = checkInitRepository(repoPath); err != nil {
 		return repo, err
 	}

-	for _, v := range lfsObjects {
-		v.ID = 0
-		v.RepositoryID = repo.ID
-		if _, err = sess2.Insert(v); err != nil {
+	if opts.GitContent && !templateRepo.IsEmpty {
+		if err = generateRepository(sess, repo, templateRepo); err != nil {
 			return repo, err
 		}
+
+		if err = repo.updateSize(sess); err != nil {
+			return repo, fmt.Errorf("failed to update size for repository: %v", err)
+		}
+
+		if err = copyLFS(sess, repo, templateRepo); err != nil {
+			return repo, fmt.Errorf("failed to copy LFS: %v", err)
+		}
 	}

-	return repo, sess2.Commit()
+	if opts.Topics {
+		for _, topic := range templateRepo.Topics {
+			if _, err = addTopicByNameToRepo(sess, repo.ID, topic); err != nil {
+				return repo, err
+			}
+		}
+	}
+
+	return repo, sess.Commit()
 }

 // GetForks returns all the forks of the repository