Refactor "dump" sub-command (#30240)
Major changes: * Move some functions like "addReader" / "isSubDir" / "addRecursiveExclude" to a separate package, and add tests * Clarify the filename&dump type logic and add tests * Clarify the logger behavior and remove FIXME comments Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
wxiaoguang
GitHub
parent
b28d3a4218
commit
654cfd1dfb
6 changed files with 374 additions and 229 deletions
296
cmd/dump.go
296
cmd/dump.go
|
|
@ -6,14 +6,13 @@ package cmd
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
"code.gitea.io/gitea/modules/dump"
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
|
@ -25,89 +24,17 @@ import (
|
|||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
func addReader(w archiver.Writer, r io.ReadCloser, info os.FileInfo, customName string, verbose bool) error {
|
||||
if verbose {
|
||||
log.Info("Adding file %s", customName)
|
||||
}
|
||||
|
||||
return w.Write(archiver.File{
|
||||
FileInfo: archiver.FileInfo{
|
||||
FileInfo: info,
|
||||
CustomName: customName,
|
||||
},
|
||||
ReadCloser: r,
|
||||
})
|
||||
}
|
||||
|
||||
func addFile(w archiver.Writer, filePath, absPath string, verbose bool) error {
|
||||
file, err := os.Open(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
fileInfo, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, file, fileInfo, filePath, verbose)
|
||||
}
|
||||
|
||||
func isSubdir(upper, lower string) (bool, error) {
|
||||
if relPath, err := filepath.Rel(upper, lower); err != nil {
|
||||
return false, err
|
||||
} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
type outputType struct {
|
||||
Enum []string
|
||||
Default string
|
||||
selected string
|
||||
}
|
||||
|
||||
func (o outputType) Join() string {
|
||||
return strings.Join(o.Enum, ", ")
|
||||
}
|
||||
|
||||
func (o *outputType) Set(value string) error {
|
||||
for _, enum := range o.Enum {
|
||||
if enum == value {
|
||||
o.selected = value
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("allowed values are %s", o.Join())
|
||||
}
|
||||
|
||||
func (o outputType) String() string {
|
||||
if o.selected == "" {
|
||||
return o.Default
|
||||
}
|
||||
return o.selected
|
||||
}
|
||||
|
||||
var outputTypeEnum = &outputType{
|
||||
Enum: []string{"zip", "tar", "tar.sz", "tar.gz", "tar.xz", "tar.bz2", "tar.br", "tar.lz4", "tar.zst"},
|
||||
Default: "zip",
|
||||
}
|
||||
|
||||
// CmdDump represents the available dump sub-command.
|
||||
var CmdDump = &cli.Command{
|
||||
Name: "dump",
|
||||
Usage: "Dump Gitea files and database",
|
||||
Description: `Dump compresses all related files and database into zip file.
|
||||
It can be used for backup and capture Gitea server image to send to maintainer`,
|
||||
Action: runDump,
|
||||
Name: "dump",
|
||||
Usage: "Dump Gitea files and database",
|
||||
Description: `Dump compresses all related files and database into zip file. It can be used for backup and capture Gitea server image to send to maintainer`,
|
||||
Action: runDump,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "file",
|
||||
Aliases: []string{"f"},
|
||||
Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
|
||||
Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
|
||||
Usage: `Name of the dump file which will be created, default to "gitea-dump-{time}.zip". Supply '-' for stdout. See type for available types.`,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "verbose",
|
||||
|
|
@ -160,64 +87,52 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
|
|||
Name: "skip-index",
|
||||
Usage: "Skip bleve index data",
|
||||
},
|
||||
&cli.GenericFlag{
|
||||
&cli.StringFlag{
|
||||
Name: "type",
|
||||
Value: outputTypeEnum,
|
||||
Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
|
||||
Usage: fmt.Sprintf(`Dump output format, default to "zip", supported types: %s`, strings.Join(dump.SupportedOutputTypes, ", ")),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func fatal(format string, args ...any) {
|
||||
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
||||
log.Fatal(format, args...)
|
||||
}
|
||||
|
||||
func runDump(ctx *cli.Context) error {
|
||||
var file *os.File
|
||||
fileName := ctx.String("file")
|
||||
outType := ctx.String("type")
|
||||
if fileName == "-" {
|
||||
file = os.Stdout
|
||||
setupConsoleLogger(log.FATAL, log.CanColorStderr, os.Stderr)
|
||||
} else {
|
||||
for _, suffix := range outputTypeEnum.Enum {
|
||||
if strings.HasSuffix(fileName, "."+suffix) {
|
||||
fileName = strings.TrimSuffix(fileName, "."+suffix)
|
||||
break
|
||||
}
|
||||
}
|
||||
fileName += "." + outType
|
||||
}
|
||||
setting.MustInstalled()
|
||||
|
||||
// make sure we are logging to the console no matter what the configuration tells us do to
|
||||
// FIXME: don't use CfgProvider directly
|
||||
if _, err := setting.CfgProvider.Section("log").NewKey("MODE", "console"); err != nil {
|
||||
fatal("Setting logging mode to console failed: %v", err)
|
||||
}
|
||||
if _, err := setting.CfgProvider.Section("log.console").NewKey("STDERR", "true"); err != nil {
|
||||
fatal("Setting console logger to stderr failed: %v", err)
|
||||
}
|
||||
|
||||
// Set loglevel to Warn if quiet-mode is requested
|
||||
if ctx.Bool("quiet") {
|
||||
if _, err := setting.CfgProvider.Section("log.console").NewKey("LEVEL", "Warn"); err != nil {
|
||||
fatal("Setting console log-level failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if !setting.InstallLock {
|
||||
log.Error("Is '%s' really the right config path?\n", setting.CustomConf)
|
||||
return fmt.Errorf("gitea is not initialized")
|
||||
}
|
||||
setting.LoadSettings() // cannot access session settings otherwise
|
||||
|
||||
quite := ctx.Bool("quiet")
|
||||
verbose := ctx.Bool("verbose")
|
||||
if verbose && ctx.Bool("quiet") {
|
||||
return fmt.Errorf("--quiet and --verbose cannot both be set")
|
||||
if verbose && quite {
|
||||
fatal("Option --quiet and --verbose cannot both be set")
|
||||
}
|
||||
|
||||
// outFileName is either "-" or a file name (will be made absolute)
|
||||
outFileName, outType := dump.PrepareFileNameAndType(ctx.String("file"), ctx.String("type"))
|
||||
if outType == "" {
|
||||
fatal("Invalid output type")
|
||||
}
|
||||
|
||||
outFile := os.Stdout
|
||||
if outFileName != "-" {
|
||||
var err error
|
||||
if outFileName, err = filepath.Abs(outFileName); err != nil {
|
||||
fatal("Unable to get absolute path of dump file: %v", err)
|
||||
}
|
||||
if exist, _ := util.IsExist(outFileName); exist {
|
||||
fatal("Dump file %q exists", outFileName)
|
||||
}
|
||||
if outFile, err = os.Create(outFileName); err != nil {
|
||||
fatal("Unable to create dump file %q: %v", outFileName, err)
|
||||
}
|
||||
defer outFile.Close()
|
||||
}
|
||||
|
||||
setupConsoleLogger(util.Iif(quite, log.WARN, log.INFO), log.CanColorStderr, os.Stderr)
|
||||
|
||||
setting.DisableLoggerInit()
|
||||
setting.LoadSettings() // cannot access session settings otherwise
|
||||
|
||||
stdCtx, cancel := installSignals()
|
||||
defer cancel()
|
||||
|
||||
|
|
@ -226,44 +141,32 @@ func runDump(ctx *cli.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
if err := storage.Init(); err != nil {
|
||||
if err = storage.Init(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if file == nil {
|
||||
file, err = os.Create(fileName)
|
||||
if err != nil {
|
||||
fatal("Unable to open %s: %v", fileName, err)
|
||||
}
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
absFileName, err := filepath.Abs(fileName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var iface any
|
||||
if fileName == "-" {
|
||||
iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
|
||||
} else {
|
||||
iface, err = archiver.ByExtension(fileName)
|
||||
}
|
||||
archiverGeneric, err := archiver.ByExtension("." + outType)
|
||||
if err != nil {
|
||||
fatal("Unable to get archiver for extension: %v", err)
|
||||
}
|
||||
|
||||
w, _ := iface.(archiver.Writer)
|
||||
if err := w.Create(file); err != nil {
|
||||
archiverWriter := archiverGeneric.(archiver.Writer)
|
||||
if err := archiverWriter.Create(outFile); err != nil {
|
||||
fatal("Creating archiver.Writer failed: %v", err)
|
||||
}
|
||||
defer w.Close()
|
||||
defer archiverWriter.Close()
|
||||
|
||||
dumper := &dump.Dumper{
|
||||
Writer: archiverWriter,
|
||||
Verbose: verbose,
|
||||
}
|
||||
dumper.GlobalExcludeAbsPath(outFileName)
|
||||
|
||||
if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
|
||||
log.Info("Skip dumping local repositories")
|
||||
} else {
|
||||
log.Info("Dumping local repositories... %s", setting.RepoRootPath)
|
||||
if err := addRecursiveExclude(w, "repos", setting.RepoRootPath, []string{absFileName}, verbose); err != nil {
|
||||
if err := dumper.AddRecursiveExclude("repos", setting.RepoRootPath, nil); err != nil {
|
||||
fatal("Failed to include repositories: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -276,8 +179,7 @@ func runDump(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, object, info, path.Join("data", "lfs", objPath), verbose)
|
||||
return dumper.AddReader(object, info, path.Join("data", "lfs", objPath))
|
||||
}); err != nil {
|
||||
fatal("Failed to dump LFS objects: %v", err)
|
||||
}
|
||||
|
|
@ -310,15 +212,13 @@ func runDump(ctx *cli.Context) error {
|
|||
fatal("Failed to dump database: %v", err)
|
||||
}
|
||||
|
||||
if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
|
||||
if err = dumper.AddFile("gitea-db.sql", dbDump.Name()); err != nil {
|
||||
fatal("Failed to include gitea-db.sql: %v", err)
|
||||
}
|
||||
|
||||
if len(setting.CustomConf) > 0 {
|
||||
log.Info("Adding custom configuration file from %s", setting.CustomConf)
|
||||
if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
|
||||
fatal("Failed to include specified app.ini: %v", err)
|
||||
}
|
||||
log.Info("Adding custom configuration file from %s", setting.CustomConf)
|
||||
if err = dumper.AddFile("app.ini", setting.CustomConf); err != nil {
|
||||
fatal("Failed to include specified app.ini: %v", err)
|
||||
}
|
||||
|
||||
if ctx.IsSet("skip-custom-dir") && ctx.Bool("skip-custom-dir") {
|
||||
|
|
@ -326,8 +226,8 @@ func runDump(ctx *cli.Context) error {
|
|||
} else {
|
||||
customDir, err := os.Stat(setting.CustomPath)
|
||||
if err == nil && customDir.IsDir() {
|
||||
if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
|
||||
if err := addRecursiveExclude(w, "custom", setting.CustomPath, []string{absFileName}, verbose); err != nil {
|
||||
if is, _ := dump.IsSubdir(setting.AppDataPath, setting.CustomPath); !is {
|
||||
if err := dumper.AddRecursiveExclude("custom", setting.CustomPath, nil); err != nil {
|
||||
fatal("Failed to include custom: %v", err)
|
||||
}
|
||||
} else {
|
||||
|
|
@ -364,8 +264,7 @@ func runDump(ctx *cli.Context) error {
|
|||
excludes = append(excludes, setting.Attachment.Storage.Path)
|
||||
excludes = append(excludes, setting.Packages.Storage.Path)
|
||||
excludes = append(excludes, setting.Log.RootPath)
|
||||
excludes = append(excludes, absFileName)
|
||||
if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
|
||||
if err := dumper.AddRecursiveExclude("data", setting.AppDataPath, excludes); err != nil {
|
||||
fatal("Failed to include data directory: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -377,8 +276,7 @@ func runDump(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, object, info, path.Join("data", "attachments", objPath), verbose)
|
||||
return dumper.AddReader(object, info, path.Join("data", "attachments", objPath))
|
||||
}); err != nil {
|
||||
fatal("Failed to dump attachments: %v", err)
|
||||
}
|
||||
|
|
@ -392,8 +290,7 @@ func runDump(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return addReader(w, object, info, path.Join("data", "packages", objPath), verbose)
|
||||
return dumper.AddReader(object, info, path.Join("data", "packages", objPath))
|
||||
}); err != nil {
|
||||
fatal("Failed to dump packages: %v", err)
|
||||
}
|
||||
|
|
@ -409,80 +306,23 @@ func runDump(ctx *cli.Context) error {
|
|||
log.Error("Unable to check if %s exists. Error: %v", setting.Log.RootPath, err)
|
||||
}
|
||||
if isExist {
|
||||
if err := addRecursiveExclude(w, "log", setting.Log.RootPath, []string{absFileName}, verbose); err != nil {
|
||||
if err := dumper.AddRecursiveExclude("log", setting.Log.RootPath, nil); err != nil {
|
||||
fatal("Failed to include log: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fileName != "-" {
|
||||
if err = w.Close(); err != nil {
|
||||
_ = util.Remove(fileName)
|
||||
fatal("Failed to save %s: %v", fileName, err)
|
||||
if outFileName == "-" {
|
||||
log.Info("Finish dumping to stdout")
|
||||
} else {
|
||||
if err = archiverWriter.Close(); err != nil {
|
||||
_ = os.Remove(outFileName)
|
||||
fatal("Failed to save %q: %v", outFileName, err)
|
||||
}
|
||||
|
||||
if err := os.Chmod(fileName, 0o600); err != nil {
|
||||
if err = os.Chmod(outFileName, 0o600); err != nil {
|
||||
log.Info("Can't change file access permissions mask to 0600: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if fileName != "-" {
|
||||
log.Info("Finish dumping in file %s", fileName)
|
||||
} else {
|
||||
log.Info("Finish dumping to stdout")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
|
||||
func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
|
||||
absPath, err := filepath.Abs(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dir, err := os.Open(absPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
|
||||
files, err := dir.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, file := range files {
|
||||
currentAbsPath := filepath.Join(absPath, file.Name())
|
||||
currentInsidePath := path.Join(insidePath, file.Name())
|
||||
if file.IsDir() {
|
||||
if !util.SliceContainsString(excludeAbsPath, currentAbsPath) {
|
||||
if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// only copy regular files and symlink regular files, skip non-regular files like socket/pipe/...
|
||||
shouldAdd := file.Mode().IsRegular()
|
||||
if !shouldAdd && file.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
target, err := filepath.EvalSymlinks(currentAbsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
targetStat, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
shouldAdd = targetStat.Mode().IsRegular()
|
||||
}
|
||||
if shouldAdd {
|
||||
if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Info("Finish dumping in file %s", outFileName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue