nune/gitea
nune/gitea
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Improve testing and try to fix MySQL hanging (#32515)

Browse source 
By some CI fine tunes (`run tests`), SQLite & MSSQL could complete
in about 12~13 minutes (before > 14), MySQL could complete in 18 minutes
(before: about 23 or even > 30)

Major changes:

1. use tmpfs for MySQL storage
1. run `make test-mysql` instead of `make integration-test-coverage`
because the code coverage is not really used at the moment.
1. refactor testlogger to make it more reliable and be able to report
stuck stacktrace
1. do not requeue failed items when a queue is being flushed (failed
items would keep failing and make flush uncompleted)
1. reduce the file sizes for testing
1. use math ChaCha20 random data instead of crypot/rand (for testing
purpose only)
1. no need to `DeleteRepository` in `TestLinguist`
1. other related refactoring to make code easier to maintain
This commit is contained in:
wxiaoguang 2024-11-15 23:45:07 +08:00 • committed by GitHub
parent a0c0cb3a2c
commit ecbb03dc6d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 201 additions and 246 deletions
Download patch file Download diff file Expand all files Collapse all files

105
modules/testlogger/testlogger.go
View file

@ -19,9 +19,10 @@ import (
)
var (
prefix string
SlowTest = 10 * time.Second
SlowFlush = 5 * time.Second
prefix string
TestTimeout = 10 * time.Minute
TestSlowRun = 10 * time.Second
TestSlowFlush = 1 * time.Second
)
var WriterCloser = &testLoggerWriterCloser{}
@ -89,79 +90,97 @@ func (w *testLoggerWriterCloser) Reset() {
w.Unlock()
}
// Printf takes a format and args and prints the string to os.Stdout
func Printf(format string, args ...any) {
if !log.CanColorStdout {
for i := 0; i < len(args); i++ {
if c, ok := args[i].(*log.ColoredValue); ok {
args[i] = c.Value()
}
}
}
_, _ = fmt.Fprintf(os.Stdout, format, args...)
}
// PrintCurrentTest prints the current test to os.Stdout
func PrintCurrentTest(t testing.TB, skip ...int) func() {
t.Helper()
start := time.Now()
runStart := time.Now()
actualSkip := util.OptionalArg(skip) + 1
_, filename, line, _ := runtime.Caller(actualSkip)
if log.CanColorStdout {
_, _ = fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line)
} else {
_, _ = fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line)
}
Printf("=== %s (%s:%d)\n", log.NewColoredValue(t.Name()), strings.TrimPrefix(filename, prefix), line)
WriterCloser.pushT(t)
return func() {
took := time.Since(start)
if took > SlowTest {
if log.CanColorStdout {
_, _ = fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow)))
} else {
_, _ = fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", t.Name(), took)
timeoutChecker := time.AfterFunc(TestTimeout, func() {
l := 128 * 1024
var stack []byte
for {
stack = make([]byte, l)
n := runtime.Stack(stack, true)
if n <= l {
stack = stack[:n]
break
}
l = n
}
timer := time.AfterFunc(SlowFlush, func() {
if log.CanColorStdout {
_, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), SlowFlush)
} else {
_, _ = fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush)
}
Printf("!!! %s ... timeout: %v ... stacktrace:\n%s\n\n", log.NewColoredValue(t.Name(), log.Bold, log.FgRed), TestTimeout, string(stack))
})
return func() {
flushStart := time.Now()
slowFlushChecker := time.AfterFunc(TestSlowFlush, func() {
Printf("+++ %s ... still flushing after %v ...\n", log.NewColoredValue(t.Name(), log.Bold, log.FgRed), TestSlowFlush)
})
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
t.Errorf("Flushing queues failed with error %v", err)
}
timer.Stop()
flushTook := time.Since(start) - took
if flushTook > SlowFlush {
if log.CanColorStdout {
_, _ = fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed)))
} else {
_, _ = fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook)
}
slowFlushChecker.Stop()
timeoutChecker.Stop()
runDuration := time.Since(runStart)
flushDuration := time.Since(flushStart)
if runDuration > TestSlowRun {
Printf("+++ %s is a slow test (run: %v, flush: %v)\n", log.NewColoredValue(t.Name(), log.Bold, log.FgYellow), runDuration, flushDuration)
}
WriterCloser.popT()
}
}
// Printf takes a format and args and prints the string to os.Stdout
func Printf(format string, args ...any) {
if log.CanColorStdout {
for i := 0; i < len(args); i++ {
args[i] = log.NewColoredValue(args[i])
}
}
_, _ = fmt.Fprintf(os.Stdout, "\t"+format, args...)
}
// TestLogEventWriter is a logger which will write to the testing log
type TestLogEventWriter struct {
*log.EventWriterBaseImpl
}
// NewTestLoggerWriter creates a TestLogEventWriter as a log.LoggerProvider
func NewTestLoggerWriter(name string, mode log.WriterMode) log.EventWriter {
// newTestLoggerWriter creates a TestLogEventWriter as a log.LoggerProvider
func newTestLoggerWriter(name string, mode log.WriterMode) log.EventWriter {
w := &TestLogEventWriter{}
w.EventWriterBaseImpl = log.NewEventWriterBase(name, "test-log-writer", mode)
w.OutputWriteCloser = WriterCloser
return w
}
func init() {
func Init() {
const relFilePath = "modules/testlogger/testlogger.go"
_, filename, _, _ := runtime.Caller(0)
if !strings.HasSuffix(filename, relFilePath) {
panic("source code file path doesn't match expected: " + relFilePath)
}
prefix = strings.TrimSuffix(filename, relFilePath)
log.RegisterEventWriter("test", newTestLoggerWriter)
duration, err := time.ParseDuration(os.Getenv("GITEA_TEST_SLOW_RUN"))
if err == nil && duration > 0 {
TestSlowRun = duration
}
duration, err = time.ParseDuration(os.Getenv("GITEA_TEST_SLOW_FLUSH"))
if err == nil && duration > 0 {
TestSlowFlush = duration
}
}
func Fatalf(format string, args ...any) {
Printf(format+"\n", args...)
os.Exit(1)
}