From 79275d9db409bd2bc6b9985f0ac0a0e1a3ec0d4e Mon Sep 17 00:00:00 2001 From: qwerty287 <80460567+qwerty287@users.noreply.github.com> Date: Wed, 26 Oct 2022 18:21:08 +0200 Subject: [PATCH 001/182] Fix 500 on PR files API (#21602) (#21607) Fixes an 500 error/panic if using the changed PR files API with pages that should return empty lists because there are no items anymore. `start-end` is then < 0 which ends in panic. Backport https://github.com/go-gitea/gitea/pull/21602 Co-authored-by: Lunny Xiao Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: delvh --- routers/api/v1/repo/pull.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/repo/pull.go b/routers/api/v1/repo/pull.go index f6507dceba..ebb9c0f261 100644 --- a/routers/api/v1/repo/pull.go +++ b/routers/api/v1/repo/pull.go @@ -1443,7 +1443,11 @@ func GetPullRequestFiles(ctx *context.APIContext) { end = totalNumberOfFiles } - apiFiles := make([]*api.ChangedFile, 0, end-start) + lenFiles := end - start + if lenFiles < 0 { + lenFiles = 0 + } + apiFiles := make([]*api.ChangedFile, 0, lenFiles) for i := start; i < end; i++ { apiFiles = append(apiFiles, convert.ToChangedFile(diff.Files[i], pr.HeadRepo, endCommitID)) } From 4869f9c3c8418a624723f5ae5e63bf893a8f80c1 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 27 Oct 2022 11:17:47 +0800 Subject: [PATCH 002/182] Revert: auto generate INTERNAL_TOKEN (#21608) (#21609) Backport #21608 Follow #19663 Some users do cluster deployment, they still depend on this auto-generating behavior. --- modules/setting/setting.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index f93be2fbd1..9c4f4ced12 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -22,6 +22,7 @@ import ( "time" "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/generate" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/user" @@ -962,6 +963,11 @@ func loadFromConf(allowEmpty bool, extraConfig string) { SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20) InternalToken = loadSecret(sec, "INTERNAL_TOKEN_URI", "INTERNAL_TOKEN") + if InstallLock && InternalToken == "" { + // if Gitea has been installed but the InternalToken hasn't been generated (upgrade from an old release), we should generate + // some users do cluster deployment, they still depend on this auto-generating behavior. + generateSaveInternalToken() + } cfgdata := sec.Key("PASSWORD_COMPLEXITY").Strings(",") if len(cfgdata) == 0 { @@ -1182,6 +1188,19 @@ func loadSecret(sec *ini.Section, uriKey, verbatimKey string) string { } } +// generateSaveInternalToken generates and saves the internal token to app.ini +func generateSaveInternalToken() { + token, err := generate.NewInternalToken() + if err != nil { + log.Fatal("Error generate internal token: %v", err) + } + + InternalToken = token + CreateOrAppendToCustomConf("security.INTERNAL_TOKEN", func(cfg *ini.File) { + cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(token) + }) +} + // MakeAbsoluteAssetURL returns the absolute asset url prefix without a trailing slash func MakeAbsoluteAssetURL(appURL, staticURLPrefix string) string { parsedPrefix, err := url.Parse(strings.TrimSuffix(staticURLPrefix, "/")) From 4d39fd8aaec48d99d845b877b94ee58763ccccab Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 27 Oct 2022 10:47:06 +0200 Subject: [PATCH 003/182] Fix `Timestamp.IsZero` (#21593) (#21603) Backport of #21593 Co-authored-by: Lunny Xiao --- modules/timeutil/timestamp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/timeutil/timestamp.go b/modules/timeutil/timestamp.go index 40fcb8603f..36b2aff575 100644 --- a/modules/timeutil/timestamp.go +++ b/modules/timeutil/timestamp.go @@ -103,5 +103,5 @@ func (ts TimeStamp) FormatDate() string { // IsZero is zero time func (ts TimeStamp) IsZero() bool { - return ts.AsTimeInLocation(time.Local).IsZero() + return int64(ts) == 0 } From 6dba648e5d94084bc37a824fe9909529e1d8dafa Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 29 Oct 2022 07:24:57 +0200 Subject: [PATCH 004/182] Use CSS color-scheme instead of invert (#21616) (#21623) Backport #21616 to 1.18 The [`color-scheme`](https://developer.mozilla.org/en-US/docs/Web/CSS/color-scheme) property changes the base color of certain form elements like the datepicker icon in Chrome. Set it and remove the previous invert hack. Before with invert removed: Screen Shot 2022-10-27 at 11 42 54 Screen Shot 2022-10-27 at 12 23 28 After: Screen Shot 2022-10-27 at 11 43 05 Screen Shot 2022-10-27 at 11 44 36 Screen Shot 2022-10-27 at 12 23 42 Co-authored-by: Lunny Xiao --- web_src/less/_base.less | 1 + web_src/less/themes/theme-arc-green.less | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web_src/less/_base.less b/web_src/less/_base.less index 6846a21ef9..f1691d8156 100644 --- a/web_src/less/_base.less +++ b/web_src/less/_base.less @@ -168,6 +168,7 @@ --color-active-line: #fffbdd; accent-color: var(--color-accent); + color-scheme: light; } :root * { diff --git a/web_src/less/themes/theme-arc-green.less b/web_src/less/themes/theme-arc-green.less index a2d2e5d082..229ffd8937 100644 --- a/web_src/less/themes/theme-arc-green.less +++ b/web_src/less/themes/theme-arc-green.less @@ -153,10 +153,9 @@ --color-accent: var(--color-primary-light-1); --color-small-accent: var(--color-primary-light-5); --color-active-line: #534d1b; -} -::-webkit-calendar-picker-indicator { - filter: invert(.8); + accent-color: var(--color-accent); + color-scheme: dark; } /* invert emojis that are hard to read otherwise */ From e4a10f8c78963d8d36a735e53fca55c193c2cc2e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Sun, 30 Oct 2022 11:17:11 +0800 Subject: [PATCH 005/182] Sync git hooks when config file path changed (#21619) (#21626) Backport #21619 . A patch to #17335. Just like AppPath, Gitea writes its own CustomConf into git hook scripts too. If Gitea's CustomConf changes, then the git push may fail. Co-authored-by: techknowlogick Co-authored-by: zeripath --- modules/system/item_runtime.go | 3 ++- routers/init.go | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/modules/system/item_runtime.go b/modules/system/item_runtime.go index ef758a5675..e022a0daad 100644 --- a/modules/system/item_runtime.go +++ b/modules/system/item_runtime.go @@ -6,7 +6,8 @@ package system // RuntimeState contains app state for runtime, and we can save remote version for update checker here in future type RuntimeState struct { - LastAppPath string `json:"last_app_path"` + LastAppPath string `json:"last_app_path"` + LastCustomConf string `json:"last_custom_conf"` } // Name returns the item name diff --git a/routers/init.go b/routers/init.go index 9045437f87..53b33f468f 100644 --- a/routers/init.go +++ b/routers/init.go @@ -76,21 +76,31 @@ func InitGitServices() { mustInit(repo_service.Init) } -func syncAppPathForGit(ctx context.Context) error { +func syncAppConfForGit(ctx context.Context) error { runtimeState := new(system.RuntimeState) if err := system.AppState.Get(runtimeState); err != nil { return err } + + updated := false if runtimeState.LastAppPath != setting.AppPath { log.Info("AppPath changed from '%s' to '%s'", runtimeState.LastAppPath, setting.AppPath) + runtimeState.LastAppPath = setting.AppPath + updated = true + } + if runtimeState.LastCustomConf != setting.CustomConf { + log.Info("CustomConf changed from '%s' to '%s'", runtimeState.LastCustomConf, setting.CustomConf) + runtimeState.LastCustomConf = setting.CustomConf + updated = true + } + if updated { log.Info("re-sync repository hooks ...") mustInitCtx(ctx, repo_service.SyncRepositoryHooks) log.Info("re-write ssh public keys ...") mustInit(asymkey_model.RewriteAllPublicKeys) - runtimeState.LastAppPath = setting.AppPath return system.AppState.Set(runtimeState) } return nil @@ -153,7 +163,7 @@ func GlobalInitInstalled(ctx context.Context) { mustInit(repo_migrations.Init) eventsource.GetManager().Init() - mustInitCtx(ctx, syncAppPathForGit) + mustInitCtx(ctx, syncAppConfForGit) mustInit(ssh.Init) From 7a8e34b255190d176d3185f0d5049fb93b18adac Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 1 Nov 2022 23:41:31 +0800 Subject: [PATCH 006/182] Deal with markdown template without metadata (#21639) (#21654) Backport #21639 . Fixed #21636. Related to #20987. A markdown template without metadata should not be treated as an invalid template. And this PR fixed another bug that non-template files(neither .md nor .yaml) are treated as yaml files. image --- modules/issue/template/unmarshal.go | 30 +++++++++++++++++++++-------- modules/structs/issue.go | 4 ++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/modules/issue/template/unmarshal.go b/modules/issue/template/unmarshal.go index e695d1e1cc..24587b0fed 100644 --- a/modules/issue/template/unmarshal.go +++ b/modules/issue/template/unmarshal.go @@ -14,6 +14,7 @@ import ( "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" "gopkg.in/yaml.v2" ) @@ -95,14 +96,27 @@ func unmarshal(filename string, content []byte) (*api.IssueTemplate, error) { }{} if typ := it.Type(); typ == api.IssueTemplateTypeMarkdown { - templateBody, err := markdown.ExtractMetadata(string(content), it) - if err != nil { - return nil, err - } - it.Content = templateBody - if it.About == "" { - if _, err := markdown.ExtractMetadata(string(content), compatibleTemplate); err == nil && compatibleTemplate.About != "" { - it.About = compatibleTemplate.About + if templateBody, err := markdown.ExtractMetadata(string(content), it); err != nil { + // The only thing we know here is that we can't extract metadata from the content, + // it's hard to tell if metadata doesn't exist or metadata isn't valid. + // There's an example template: + // + // --- + // # Title + // --- + // Content + // + // It could be a valid markdown with two horizontal lines, or an invalid markdown with wrong metadata. + + it.Content = string(content) + it.Name = filepath.Base(it.FileName) + it.About, _ = util.SplitStringAtByteN(it.Content, 80) + } else { + it.Content = templateBody + if it.About == "" { + if _, err := markdown.ExtractMetadata(string(content), compatibleTemplate); err == nil && compatibleTemplate.About != "" { + it.About = compatibleTemplate.About + } } } } else if typ == api.IssueTemplateTypeYaml { diff --git a/modules/structs/issue.go b/modules/structs/issue.go index 27ec81f728..70f5e1ba8e 100644 --- a/modules/structs/issue.go +++ b/modules/structs/issue.go @@ -170,7 +170,7 @@ func (it IssueTemplate) Type() IssueTemplateType { if ext := filepath.Ext(it.FileName); ext == ".md" { return IssueTemplateTypeMarkdown } else if ext == ".yaml" || ext == ".yml" { - return "yaml" + return IssueTemplateTypeYaml } - return IssueTemplateTypeYaml + return "" } From fd4e7447e7ef80ef397dbf91ae4bf3a3d5c6727f Mon Sep 17 00:00:00 2001 From: silverwind Date: Tue, 1 Nov 2022 23:31:17 +0100 Subject: [PATCH 007/182] Fix opaque background on mermaid diagrams (#21642) (#21652) Backport #21642 Browsers introduce a opaque background on iframes if the iframe element's color-scheme does not match the document's color scheme which in case of a dark theme results in a mismatch and the browser adds a white background. Avoid this by specifying the same color scheme outside and inside the iframe. See https://fvsch.com/transparent-iframes for more info. My initial attempt was to make the iframe document the same color-scheme as the parent page (light or dark) but with that, there was a ugly background flash on load in Chrome because Chrome apparently always loads iframe in light scheme initially. Firefox still shows a background flash on load but this is not possible to get rid of and it's certainly a browser bug. Before: Screen Shot 2022-10-31 at 13 30 55 After: Screen Shot 2022-10-31 at 13 30 36 --- web_src/js/markup/mermaid.js | 1 + web_src/less/markup/content.less | 1 + 2 files changed, 2 insertions(+) diff --git a/web_src/js/markup/mermaid.js b/web_src/js/markup/mermaid.js index 62de9a3aae..984946045d 100644 --- a/web_src/js/markup/mermaid.js +++ b/web_src/js/markup/mermaid.js @@ -2,6 +2,7 @@ import {isDarkTheme} from '../utils.js'; const {mermaidMaxSourceCharacters} = window.config; const iframeCss = ` + :root {color-scheme: normal} body {margin: 0; padding: 0} #mermaid {display: block; margin: 0 auto} `; diff --git a/web_src/less/markup/content.less b/web_src/less/markup/content.less index 7ee55ea7f8..80c6267af8 100644 --- a/web_src/less/markup/content.less +++ b/web_src/less/markup/content.less @@ -537,6 +537,7 @@ width: 100%; height: var(--height-loading); // actual height is set in JS after loading overflow: hidden; + color-scheme: normal; // match the value inside the iframe to allow it to become transparent } .markup-block-error { From 3a0d000b947ff9a1fcdc90698e6d9bb852a67882 Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 1 Nov 2022 22:32:03 +0000 Subject: [PATCH 008/182] Fix repository adoption on Windows (#21646) (#21650) Backport #21646 A bug was introduced in #17865 where filepath.Join is used to join putative unadopted repository owner and names together. This is incorrect as these names are then used as repository names - which shoud have the '/' separator. This means that adoption will not work on Windows servers. Fix #21632 Signed-off-by: Andrew Thornton --- services/repository/adopt.go | 11 ++++++----- services/repository/adopt_test.go | 8 ++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/services/repository/adopt.go b/services/repository/adopt.go index 3b986c66c6..a9a0639548 100644 --- a/services/repository/adopt.go +++ b/services/repository/adopt.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "os" + "path" "path/filepath" "strings" @@ -218,21 +219,21 @@ func DeleteUnadoptedRepository(doer, u *user_model.User, repoName string) error return util.RemoveAll(repoPath) } -type unadoptedRrepositories struct { +type unadoptedRepositories struct { repositories []string index int start int end int } -func (unadopted *unadoptedRrepositories) add(repository string) { +func (unadopted *unadoptedRepositories) add(repository string) { if unadopted.index >= unadopted.start && unadopted.index < unadopted.end { unadopted.repositories = append(unadopted.repositories, repository) } unadopted.index++ } -func checkUnadoptedRepositories(userName string, repoNamesToCheck []string, unadopted *unadoptedRrepositories) error { +func checkUnadoptedRepositories(userName string, repoNamesToCheck []string, unadopted *unadoptedRepositories) error { if len(repoNamesToCheck) == 0 { return nil } @@ -264,7 +265,7 @@ func checkUnadoptedRepositories(userName string, repoNamesToCheck []string, unad } for _, repoName := range repoNamesToCheck { if !repoNames.Contains(repoName) { - unadopted.add(filepath.Join(userName, repoName)) + unadopted.add(path.Join(userName, repoName)) // These are not used as filepaths - but as reponames - therefore use path.Join not filepath.Join } } return nil @@ -292,7 +293,7 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in var repoNamesToCheck []string start := (opts.Page - 1) * opts.PageSize - unadopted := &unadoptedRrepositories{ + unadopted := &unadoptedRepositories{ repositories: make([]string, 0, opts.PageSize), start: start, end: start + opts.PageSize, diff --git a/services/repository/adopt_test.go b/services/repository/adopt_test.go index 685bfe9bc4..b450005f34 100644 --- a/services/repository/adopt_test.go +++ b/services/repository/adopt_test.go @@ -19,7 +19,7 @@ import ( func TestCheckUnadoptedRepositories_Add(t *testing.T) { start := 10 end := 20 - unadopted := &unadoptedRrepositories{ + unadopted := &unadoptedRepositories{ start: start, end: end, index: 0, @@ -39,7 +39,7 @@ func TestCheckUnadoptedRepositories(t *testing.T) { // // Non existent user // - unadopted := &unadoptedRrepositories{start: 0, end: 100} + unadopted := &unadoptedRepositories{start: 0, end: 100} err := checkUnadoptedRepositories("notauser", []string{"repo"}, unadopted) assert.NoError(t, err) assert.Equal(t, 0, len(unadopted.repositories)) @@ -50,14 +50,14 @@ func TestCheckUnadoptedRepositories(t *testing.T) { userName := "user2" repoName := "repo2" unadoptedRepoName := "unadopted" - unadopted = &unadoptedRrepositories{start: 0, end: 100} + unadopted = &unadoptedRepositories{start: 0, end: 100} err = checkUnadoptedRepositories(userName, []string{repoName, unadoptedRepoName}, unadopted) assert.NoError(t, err) assert.Equal(t, []string{path.Join(userName, unadoptedRepoName)}, unadopted.repositories) // // Existing (adopted) repository is not returned // - unadopted = &unadoptedRrepositories{start: 0, end: 100} + unadopted = &unadoptedRepositories{start: 0, end: 100} err = checkUnadoptedRepositories(userName, []string{repoName}, unadopted) assert.NoError(t, err) assert.Equal(t, 0, len(unadopted.repositories)) From 89d52922d0cec319430e67add29aa4f5a54b1d03 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 4 Nov 2022 04:54:25 +0800 Subject: [PATCH 009/182] Fix token generation when using INTERNAL_TOKEN_URI (#21669) (#21670) Backport #21669 Fix https://github.com/go-gitea/gitea/issues/21666 Caused by https://github.com/go-gitea/gitea/pull/19663 Before: when install, the INTERNAL_TOKEN was always generated and saved. But the internal token may be already there by INTERNAL_TOKEN_URI After: INTERNAL_TOKEN_URI file must be non-empty. When install, skip internal token generation if the token exists. --- modules/setting/setting.go | 12 +++++++++++- routers/install/install.go | 14 +++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 9c4f4ced12..bd035403c8 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1156,6 +1156,8 @@ func parseAuthorizedPrincipalsAllow(values []string) ([]string, bool) { return authorizedPrincipalsAllow, true } +// loadSecret load the secret from ini by uriKey or verbatimKey, only one of them could be set +// If the secret is loaded from uriKey (file), the file should be non-empty, to guarantee the behavior stable and clear. func loadSecret(sec *ini.Section, uriKey, verbatimKey string) string { // don't allow setting both URI and verbatim string uri := sec.Key(uriKey).String() @@ -1179,7 +1181,15 @@ func loadSecret(sec *ini.Section, uriKey, verbatimKey string) string { if err != nil { log.Fatal("Failed to read %s (%s): %v", uriKey, tempURI.RequestURI(), err) } - return strings.TrimSpace(string(buf)) + val := strings.TrimSpace(string(buf)) + if val == "" { + // The file shouldn't be empty, otherwise we can not know whether the user has ever set the KEY or KEY_URI + // For example: if INTERNAL_TOKEN_URI=file:///empty-file, + // Then if the token is re-generated during installation and saved to INTERNAL_TOKEN + // Then INTERNAL_TOKEN and INTERNAL_TOKEN_URI both exist, that's a fatal error (they shouldn't) + log.Fatal("Failed to read %s (%s): the file is empty", uriKey, tempURI.RequestURI()) + } + return val // only file URIs are allowed default: diff --git a/routers/install/install.go b/routers/install/install.go index 962dee8c86..fb5401559b 100644 --- a/routers/install/install.go +++ b/routers/install/install.go @@ -473,12 +473,16 @@ func SubmitInstall(ctx *context.Context) { cfg.Section("security").Key("INSTALL_LOCK").SetValue("true") - var internalToken string - if internalToken, err = generate.NewInternalToken(); err != nil { - ctx.RenderWithErr(ctx.Tr("install.internal_token_failed", err), tplInstall, &form) - return + // the internal token could be read from INTERNAL_TOKEN or INTERNAL_TOKEN_URI (the file is guaranteed to be non-empty) + // if there is no InternalToken, generate one and save to security.INTERNAL_TOKEN + if setting.InternalToken == "" { + var internalToken string + if internalToken, err = generate.NewInternalToken(); err != nil { + ctx.RenderWithErr(ctx.Tr("install.internal_token_failed", err), tplInstall, &form) + return + } + cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(internalToken) } - cfg.Section("security").Key("INTERNAL_TOKEN").SetValue(internalToken) // if there is already a SECRET_KEY, we should not overwrite it, otherwise the encrypted data will not be able to be decrypted if setting.SecretKey == "" { From e054f80fe0e6c6cc2e6a1338abb0c9e260e4d8be Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 8 Nov 2022 19:10:06 -0600 Subject: [PATCH 010/182] Allow local package identifiers for PyPI packages (#21690) (#21727) Backport (#21690) Fixes #21683 Co-authored-by: Lunny Xiao Co-authored-by: KN4CK3R --- routers/api/packages/pypi/pypi.go | 11 +++++++++-- tests/integration/api_packages_pypi_test.go | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 3a046abe18..66380d832c 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -25,8 +25,15 @@ import ( var normalizer = strings.NewReplacer(".", "-", "_", "-") var nameMatcher = regexp.MustCompile(`\A[a-zA-Z0-9\.\-_]+\z`) -// https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions -var versionMatcher = regexp.MustCompile(`^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$`) +// https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions +var versionMatcher = regexp.MustCompile(`\Av?` + + `(?:[0-9]+!)?` + // epoch + `[0-9]+(?:\.[0-9]+)*` + // release segment + `(?:[-_\.]?(?:a|b|c|rc|alpha|beta|pre|preview)[-_\.]?[0-9]*)?` + // pre-release + `(?:-[0-9]+|[-_\.]?(?:post|rev|r)[-_\.]?[0-9]*)?` + // post release + `(?:[-_\.]?dev[-_\.]?[0-9]*)?` + // dev release + `(?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)?` + // local version + `\z`) func apiError(ctx *context.Context, status int, obj interface{}) { helper.LogAndProcessError(ctx, status, obj, func(message string) { diff --git a/tests/integration/api_packages_pypi_test.go b/tests/integration/api_packages_pypi_test.go index 32b3304ca7..0cd6ff7d13 100644 --- a/tests/integration/api_packages_pypi_test.go +++ b/tests/integration/api_packages_pypi_test.go @@ -29,7 +29,7 @@ func TestPackagePyPI(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) packageName := "test-package" - packageVersion := "1.0.1" + packageVersion := "1.0.1+r1234" packageAuthor := "KN4CK3R" packageDescription := "Test Description" @@ -164,7 +164,7 @@ func TestPackagePyPI(t *testing.T) { nodes := htmlDoc.doc.Find("a").Nodes assert.Len(t, nodes, 2) - hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256-%s`, root, packageName, packageVersion, hashSHA256)) + hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256-%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256)) for _, a := range nodes { for _, att := range a.Attr { From 69a54545a8915900b78fa1b7efa1252b37d1d628 Mon Sep 17 00:00:00 2001 From: Xinyu Zhou Date: Wed, 9 Nov 2022 21:19:44 +0800 Subject: [PATCH 011/182] Quick fixes monaco-editor error: "vs.editor.nullLanguage" (#21734) (#21738) Backport #21734 fixes: https://github.com/go-gitea/gitea/issues/21733 Uncaught Error: Language id "vs.editor.nullLanguage" is not configured nor known Note that this monaco-editor worked fine on 0.33.0 and broke on 0.34.0. If upstream fixed, remove this code. Signed-off-by: Xinyu Zhou Co-authored-by: Lunny Xiao --- web_src/js/bootstrap.js | 3 --- web_src/js/features/codeeditor.js | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/web_src/js/bootstrap.js b/web_src/js/bootstrap.js index b5db398545..54b7c62887 100644 --- a/web_src/js/bootstrap.js +++ b/web_src/js/bootstrap.js @@ -26,9 +26,6 @@ function processWindowErrorEvent(e) { return; // ignore such nonsense error event } - // Wait for upstream fix: https://github.com/microsoft/monaco-editor/issues/2962 - if (e.message.includes('Language id "vs.editor.nullLanguage" is not configured nor known')) return; - showGlobalErrorMessage(`JavaScript error: ${e.message} (${e.filename} @ ${e.lineno}:${e.colno}). Open browser console to see more details.`); } diff --git a/web_src/js/features/codeeditor.js b/web_src/js/features/codeeditor.js index 0366afc2c0..23a26ba2b0 100644 --- a/web_src/js/features/codeeditor.js +++ b/web_src/js/features/codeeditor.js @@ -99,6 +99,10 @@ export async function createMonaco(textarea, filename, editorOpts) { } }); + // Quick fix: https://github.com/microsoft/monaco-editor/issues/2962 + monaco.languages.register({id: 'vs.editor.nullLanguage'}); + monaco.languages.setLanguageConfiguration('vs.editor.nullLanguage', {}); + const editor = monaco.editor.create(container, { value: textarea.value, theme: 'gitea', From 805a14cc91cf1d5ddeb2c23ff1c4176c8badb5c1 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Wed, 9 Nov 2022 09:02:11 -0600 Subject: [PATCH 012/182] Remove semver compatible flag and change pypi to an array of test cases (#21708) (#21730) Backport (#21708) This addresses #21707 and adds a second package test case for a non-semver compatible version (this might be overkill though since you could also edit the old package version to have an epoch in front and see the error, this just seemed more flexible for the future). Co-authored-by: KN4CK3R --- routers/api/packages/pypi/pypi.go | 12 ++++--- routers/api/packages/pypi/pypi_test.go | 39 +++++++++++++++++++++ tests/integration/api_packages_pypi_test.go | 6 ++-- 3 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 routers/api/packages/pypi/pypi_test.go diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 66380d832c..4c8041c30c 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -21,9 +21,9 @@ import ( packages_service "code.gitea.io/gitea/services/packages" ) -// https://www.python.org/dev/peps/pep-0503/#normalized-names +// https://peps.python.org/pep-0426/#name var normalizer = strings.NewReplacer(".", "-", "_", "-") -var nameMatcher = regexp.MustCompile(`\A[a-zA-Z0-9\.\-_]+\z`) +var nameMatcher = regexp.MustCompile(`\A(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\.\-_]*[a-zA-Z0-9])\z`) // https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions var versionMatcher = regexp.MustCompile(`\Av?` + @@ -128,7 +128,7 @@ func UploadPackageFile(ctx *context.Context) { packageName := normalizer.Replace(ctx.Req.FormValue("name")) packageVersion := ctx.Req.FormValue("version") - if !nameMatcher.MatchString(packageName) || !versionMatcher.MatchString(packageVersion) { + if !isValidNameAndVersion(packageName, packageVersion) { apiError(ctx, http.StatusBadRequest, "invalid name or version") return } @@ -146,7 +146,7 @@ func UploadPackageFile(ctx *context.Context) { Name: packageName, Version: packageVersion, }, - SemverCompatible: true, + SemverCompatible: false, Creator: ctx.Doer, Metadata: &pypi_module.Metadata{ Author: ctx.Req.FormValue("author"), @@ -177,3 +177,7 @@ func UploadPackageFile(ctx *context.Context) { ctx.Status(http.StatusCreated) } + +func isValidNameAndVersion(packageName, packageVersion string) bool { + return nameMatcher.MatchString(packageName) && versionMatcher.MatchString(packageVersion) +} diff --git a/routers/api/packages/pypi/pypi_test.go b/routers/api/packages/pypi/pypi_test.go new file mode 100644 index 0000000000..56e327a347 --- /dev/null +++ b/routers/api/packages/pypi/pypi_test.go @@ -0,0 +1,39 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pypi + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsValidNameAndVersion(t *testing.T) { + // The test cases below were created from the following Python PEPs: + // https://peps.python.org/pep-0426/#name + // https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions + + // Valid Cases + assert.True(t, isValidNameAndVersion("A", "1.0.1")) + assert.True(t, isValidNameAndVersion("Test.Name.1234", "1.0.1")) + assert.True(t, isValidNameAndVersion("test_name", "1.0.1")) + assert.True(t, isValidNameAndVersion("test-name", "1.0.1")) + assert.True(t, isValidNameAndVersion("test-name", "v1.0.1")) + assert.True(t, isValidNameAndVersion("test-name", "2012.4")) + assert.True(t, isValidNameAndVersion("test-name", "1.0.1-alpha")) + assert.True(t, isValidNameAndVersion("test-name", "1.0.1a1")) + assert.True(t, isValidNameAndVersion("test-name", "1.0b2.r345.dev456")) + assert.True(t, isValidNameAndVersion("test-name", "1!1.0.1")) + assert.True(t, isValidNameAndVersion("test-name", "1.0.1+local.1")) + + // Invalid Cases + assert.False(t, isValidNameAndVersion(".test-name", "1.0.1")) + assert.False(t, isValidNameAndVersion("test!name", "1.0.1")) + assert.False(t, isValidNameAndVersion("-test-name", "1.0.1")) + assert.False(t, isValidNameAndVersion("test-name-", "1.0.1")) + assert.False(t, isValidNameAndVersion("test-name", "a1.0.1")) + assert.False(t, isValidNameAndVersion("test-name", "1.0.1aa")) + assert.False(t, isValidNameAndVersion("test-name", "1.0.0-alpha.beta")) +} diff --git a/tests/integration/api_packages_pypi_test.go b/tests/integration/api_packages_pypi_test.go index 0cd6ff7d13..83719dcca0 100644 --- a/tests/integration/api_packages_pypi_test.go +++ b/tests/integration/api_packages_pypi_test.go @@ -29,7 +29,7 @@ func TestPackagePyPI(t *testing.T) { user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) packageName := "test-package" - packageVersion := "1.0.1+r1234" + packageVersion := "1!1.0.1+r1234" packageAuthor := "KN4CK3R" packageDescription := "Test Description" @@ -72,7 +72,7 @@ func TestPackagePyPI(t *testing.T) { pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) - assert.NotNil(t, pd.SemVer) + assert.Nil(t, pd.SemVer) assert.IsType(t, &pypi.Metadata{}, pd.Metadata) assert.Equal(t, packageName, pd.Package.Name) assert.Equal(t, packageVersion, pd.Version.Version) @@ -100,7 +100,7 @@ func TestPackagePyPI(t *testing.T) { pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) - assert.NotNil(t, pd.SemVer) + assert.Nil(t, pd.SemVer) assert.IsType(t, &pypi.Metadata{}, pd.Metadata) assert.Equal(t, packageName, pd.Package.Name) assert.Equal(t, packageVersion, pd.Version.Version) From a2a42cd5deb11bc27080ffc4259d6907f0e38061 Mon Sep 17 00:00:00 2001 From: Xinyu Zhou Date: Thu, 10 Nov 2022 10:14:32 +0800 Subject: [PATCH 013/182] Fix UI language switching bug (#21597) (#21749) Backport #21597 Related: * https://github.com/go-gitea/gitea/pull/21596#issuecomment-1291450224 There was a bug when switching language by AJAX: the irrelevant POST requests were processed by the target page's handler. Now, use GET instead of POST. The GET requests should be harmless. Co-authored-by: delvh Co-authored-by: Jason Song Co-authored-by: Lunny Xiao Co-authored-by: wxiaoguang --- web_src/js/features/common-global.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/features/common-global.js b/web_src/js/features/common-global.js index a3aebc0246..b00b4aea9c 100644 --- a/web_src/js/features/common-global.js +++ b/web_src/js/features/common-global.js @@ -37,7 +37,7 @@ export function initHeadNavbarContentToggle() { export function initFootLanguageMenu() { function linkLanguageAction() { const $this = $(this); - $.post($this.data('url')).always(() => { + $.get($this.data('url')).always(() => { window.location.reload(); }); } From b9dcf991b963115f59f9cd41e6fcae09b0e9d235 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 10 Nov 2022 19:41:44 +0800 Subject: [PATCH 014/182] Fix dashboard ignored system setting cache (#21621) (#21759) backport #21621 This is a performance regression from #18058 Signed-off-by: Andrew Thornton Co-authored-by: Andrew Thornton --- models/avatars/avatar.go | 11 ++- models/system/setting.go | 40 ++++++-- models/system/setting_key.go | 5 + models/user/avatar.go | 6 +- models/user/setting.go | 36 +++++++- models/user/setting_test.go | 2 +- modules/activitypub/user_settings.go | 2 +- modules/cache/cache.go | 131 +++++++++++++++++---------- modules/system/setting.go | 46 ---------- modules/system/user_setting.go | 34 ------- modules/templates/helper.go | 3 +- routers/web/admin/config.go | 7 +- 12 files changed, 172 insertions(+), 151 deletions(-) delete mode 100644 modules/system/setting.go delete mode 100644 modules/system/user_setting.go diff --git a/models/avatars/avatar.go b/models/avatars/avatar.go index 418e9b9ccc..ec3b611312 100644 --- a/models/avatars/avatar.go +++ b/models/avatars/avatar.go @@ -150,10 +150,11 @@ func generateEmailAvatarLink(email string, size int, final bool) string { return DefaultAvatarLink() } - enableFederatedAvatar, _ := system_model.GetSetting(system_model.KeyPictureEnableFederatedAvatar) + enableFederatedAvatarSetting, _ := system_model.GetSetting(system_model.KeyPictureEnableFederatedAvatar) + enableFederatedAvatar := enableFederatedAvatarSetting.GetValueBool() var err error - if enableFederatedAvatar != nil && enableFederatedAvatar.GetValueBool() && system_model.LibravatarService != nil { + if enableFederatedAvatar && system_model.LibravatarService != nil { emailHash := saveEmailHash(email) if final { // for final link, we can spend more time on slow external query @@ -171,8 +172,10 @@ func generateEmailAvatarLink(email string, size int, final bool) string { return urlStr } - disableGravatar, _ := system_model.GetSetting(system_model.KeyPictureDisableGravatar) - if disableGravatar != nil && !disableGravatar.GetValueBool() { + disableGravatarSetting, _ := system_model.GetSetting(system_model.KeyPictureDisableGravatar) + + disableGravatar := disableGravatarSetting.GetValueBool() + if !disableGravatar { // copy GravatarSourceURL, because we will modify its Path. avatarURLCopy := *system_model.GravatarSourceURL avatarURLCopy.Path = path.Join(avatarURLCopy.Path, HashEmail(email)) diff --git a/models/system/setting.go b/models/system/setting.go index 9711d38f3b..b4011b1b3e 100644 --- a/models/system/setting.go +++ b/models/system/setting.go @@ -12,6 +12,7 @@ import ( "strings" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -35,6 +36,10 @@ func (s *Setting) TableName() string { } func (s *Setting) GetValueBool() bool { + if s == nil { + return false + } + b, _ := strconv.ParseBool(s.SettingValue) return b } @@ -75,8 +80,8 @@ func IsErrDataExpired(err error) bool { return ok } -// GetSetting returns specific setting -func GetSetting(key string) (*Setting, error) { +// GetSettingNoCache returns specific setting without using the cache +func GetSettingNoCache(key string) (*Setting, error) { v, err := GetSettings([]string{key}) if err != nil { return nil, err @@ -87,6 +92,24 @@ func GetSetting(key string) (*Setting, error) { return v[key], nil } +// GetSetting returns the setting value via the key +func GetSetting(key string) (*Setting, error) { + return cache.Get(genSettingCacheKey(key), func() (*Setting, error) { + res, err := GetSettingNoCache(key) + if err != nil { + return nil, err + } + return res, nil + }) +} + +// GetSettingBool return bool value of setting, +// none existing keys and errors are ignored and result in false +func GetSettingBool(key string) bool { + s, _ := GetSetting(key) + return s.GetValueBool() +} + // GetSettings returns specific settings func GetSettings(keys []string) (map[string]*Setting, error) { for i := 0; i < len(keys); i++ { @@ -139,12 +162,13 @@ func GetAllSettings() (AllSettings, error) { // DeleteSetting deletes a specific setting for a user func DeleteSetting(setting *Setting) error { + cache.Remove(genSettingCacheKey(setting.SettingKey)) _, err := db.GetEngine(db.DefaultContext).Delete(setting) return err } func SetSettingNoVersion(key, value string) error { - s, err := GetSetting(key) + s, err := GetSettingNoCache(key) if IsErrSettingIsNotExist(err) { return SetSetting(&Setting{ SettingKey: key, @@ -160,9 +184,13 @@ func SetSettingNoVersion(key, value string) error { // SetSetting updates a users' setting for a specific key func SetSetting(setting *Setting) error { - if err := upsertSettingValue(strings.ToLower(setting.SettingKey), setting.SettingValue, setting.Version); err != nil { + _, err := cache.Set(genSettingCacheKey(setting.SettingKey), func() (*Setting, error) { + return setting, upsertSettingValue(strings.ToLower(setting.SettingKey), setting.SettingValue, setting.Version) + }) + if err != nil { return err } + setting.Version++ return nil } @@ -213,7 +241,7 @@ var ( func Init() error { var disableGravatar bool - disableGravatarSetting, err := GetSetting(KeyPictureDisableGravatar) + disableGravatarSetting, err := GetSettingNoCache(KeyPictureDisableGravatar) if IsErrSettingIsNotExist(err) { disableGravatar = setting.GetDefaultDisableGravatar() disableGravatarSetting = &Setting{SettingValue: strconv.FormatBool(disableGravatar)} @@ -224,7 +252,7 @@ func Init() error { } var enableFederatedAvatar bool - enableFederatedAvatarSetting, err := GetSetting(KeyPictureEnableFederatedAvatar) + enableFederatedAvatarSetting, err := GetSettingNoCache(KeyPictureEnableFederatedAvatar) if IsErrSettingIsNotExist(err) { enableFederatedAvatar = setting.GetDefaultEnableFederatedAvatar(disableGravatar) enableFederatedAvatarSetting = &Setting{SettingValue: strconv.FormatBool(enableFederatedAvatar)} diff --git a/models/system/setting_key.go b/models/system/setting_key.go index 5a6ea6ed72..14105b89d0 100644 --- a/models/system/setting_key.go +++ b/models/system/setting_key.go @@ -9,3 +9,8 @@ const ( KeyPictureDisableGravatar = "picture.disable_gravatar" KeyPictureEnableFederatedAvatar = "picture.enable_federated_avatar" ) + +// genSettingCacheKey returns the cache key for some configuration +func genSettingCacheKey(key string) string { + return "system.setting." + key +} diff --git a/models/user/avatar.go b/models/user/avatar.go index f73ac56c5e..102206f3a2 100644 --- a/models/user/avatar.go +++ b/models/user/avatar.go @@ -68,11 +68,9 @@ func (u *User) AvatarLinkWithSize(size int) string { useLocalAvatar := false autoGenerateAvatar := false - var disableGravatar bool disableGravatarSetting, _ := system_model.GetSetting(system_model.KeyPictureDisableGravatar) - if disableGravatarSetting != nil { - disableGravatar = disableGravatarSetting.GetValueBool() - } + + disableGravatar := disableGravatarSetting.GetValueBool() switch { case u.UseCustomAvatar: diff --git a/models/user/setting.go b/models/user/setting.go index 5fe7c2ec23..896f3c8da1 100644 --- a/models/user/setting.go +++ b/models/user/setting.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/cache" "xorm.io/builder" ) @@ -47,9 +48,25 @@ func IsErrUserSettingIsNotExist(err error) bool { return ok } -// GetSetting returns specific setting +// genSettingCacheKey returns the cache key for some configuration +func genSettingCacheKey(userID int64, key string) string { + return fmt.Sprintf("user_%d.setting.%s", userID, key) +} + +// GetSetting returns the setting value via the key func GetSetting(uid int64, key string) (*Setting, error) { - v, err := GetUserSettings(uid, []string{key}) + return cache.Get(genSettingCacheKey(uid, key), func() (*Setting, error) { + res, err := GetSettingNoCache(uid, key) + if err != nil { + return nil, err + } + return res, nil + }) +} + +// GetSettingNoCache returns specific setting without using the cache +func GetSettingNoCache(uid int64, key string) (*Setting, error) { + v, err := GetSettings(uid, []string{key}) if err != nil { return nil, err } @@ -59,8 +76,8 @@ func GetSetting(uid int64, key string) (*Setting, error) { return v[key], nil } -// GetUserSettings returns specific settings from user -func GetUserSettings(uid int64, keys []string) (map[string]*Setting, error) { +// GetSettings returns specific settings from user +func GetSettings(uid int64, keys []string) (map[string]*Setting, error) { settings := make([]*Setting, 0, len(keys)) if err := db.GetEngine(db.DefaultContext). Where("user_id=?", uid). @@ -105,6 +122,7 @@ func GetUserSetting(userID int64, key string, def ...string) (string, error) { if err := validateUserSettingKey(key); err != nil { return "", err } + setting := &Setting{UserID: userID, SettingKey: key} has, err := db.GetEngine(db.DefaultContext).Get(setting) if err != nil { @@ -124,7 +142,10 @@ func DeleteUserSetting(userID int64, key string) error { if err := validateUserSettingKey(key); err != nil { return err } + + cache.Remove(genSettingCacheKey(userID, key)) _, err := db.GetEngine(db.DefaultContext).Delete(&Setting{UserID: userID, SettingKey: key}) + return err } @@ -133,7 +154,12 @@ func SetUserSetting(userID int64, key, value string) error { if err := validateUserSettingKey(key); err != nil { return err } - return upsertUserSettingValue(userID, key, value) + + _, err := cache.Set(genSettingCacheKey(userID, key), func() (string, error) { + return value, upsertUserSettingValue(userID, key, value) + }) + + return err } func upsertUserSettingValue(userID int64, key, value string) error { diff --git a/models/user/setting_test.go b/models/user/setting_test.go index f0083038df..5a772a8ce7 100644 --- a/models/user/setting_test.go +++ b/models/user/setting_test.go @@ -27,7 +27,7 @@ func TestSettings(t *testing.T) { assert.NoError(t, err) // get specific setting - settings, err := user_model.GetUserSettings(99, []string{keyName}) + settings, err := user_model.GetSettings(99, []string{keyName}) assert.NoError(t, err) assert.Len(t, settings, 1) assert.EqualValues(t, newSetting.SettingValue, settings[keyName].SettingValue) diff --git a/modules/activitypub/user_settings.go b/modules/activitypub/user_settings.go index fc9775b0f0..d192b9cdb2 100644 --- a/modules/activitypub/user_settings.go +++ b/modules/activitypub/user_settings.go @@ -11,7 +11,7 @@ import ( // GetKeyPair function returns a user's private and public keys func GetKeyPair(user *user_model.User) (pub, priv string, err error) { var settings map[string]*user_model.Setting - settings, err = user_model.GetUserSettings(user.ID, []string{user_model.UserActivityPubPrivPem, user_model.UserActivityPubPubPem}) + settings, err = user_model.GetSettings(user.ID, []string{user_model.UserActivityPubPrivPem, user_model.UserActivityPubPubPem}) if err != nil { return } else if len(settings) == 0 { diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 21d0cd0a04..d98b0a0cec 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -46,32 +46,64 @@ func GetCache() mc.Cache { return conn } +// Get returns the key value from cache with callback when no key exists in cache +func Get[V interface{}](key string, getFunc func() (V, error)) (V, error) { + if conn == nil || setting.CacheService.TTL == 0 { + return getFunc() + } + + cached := conn.Get(key) + if value, ok := cached.(V); ok { + return value, nil + } + + value, err := getFunc() + if err != nil { + return value, err + } + + return value, conn.Put(key, value, setting.CacheService.TTLSeconds()) +} + +// Set updates and returns the key value in the cache with callback. The old value is only removed if the updateFunc() is successful +func Set[V interface{}](key string, valueFunc func() (V, error)) (V, error) { + if conn == nil || setting.CacheService.TTL == 0 { + return valueFunc() + } + + value, err := valueFunc() + if err != nil { + return value, err + } + + return value, conn.Put(key, value, setting.CacheService.TTLSeconds()) +} + // GetString returns the key value from cache with callback when no key exists in cache func GetString(key string, getFunc func() (string, error)) (string, error) { if conn == nil || setting.CacheService.TTL == 0 { return getFunc() } - if !conn.IsExist(key) { - var ( - value string - err error - ) - if value, err = getFunc(); err != nil { + + cached := conn.Get(key) + + if cached == nil { + value, err := getFunc() + if err != nil { return value, err } - err = conn.Put(key, value, setting.CacheService.TTLSeconds()) - if err != nil { - return "", err - } + return value, conn.Put(key, value, setting.CacheService.TTLSeconds()) } - value := conn.Get(key) - if v, ok := value.(string); ok { - return v, nil + + if value, ok := cached.(string); ok { + return value, nil } - if v, ok := value.(fmt.Stringer); ok { - return v.String(), nil + + if stringer, ok := cached.(fmt.Stringer); ok { + return stringer.String(), nil } - return fmt.Sprintf("%s", conn.Get(key)), nil + + return fmt.Sprintf("%s", cached), nil } // GetInt returns key value from cache with callback when no key exists in cache @@ -79,30 +111,33 @@ func GetInt(key string, getFunc func() (int, error)) (int, error) { if conn == nil || setting.CacheService.TTL == 0 { return getFunc() } - if !conn.IsExist(key) { - var ( - value int - err error - ) - if value, err = getFunc(); err != nil { + + cached := conn.Get(key) + + if cached == nil { + value, err := getFunc() + if err != nil { return value, err } - err = conn.Put(key, value, setting.CacheService.TTLSeconds()) - if err != nil { - return 0, err - } + + return value, conn.Put(key, value, setting.CacheService.TTLSeconds()) } - switch value := conn.Get(key).(type) { + + switch v := cached.(type) { case int: - return value, nil + return v, nil case string: - v, err := strconv.Atoi(value) + value, err := strconv.Atoi(v) if err != nil { return 0, err } - return v, nil + return value, nil default: - return 0, fmt.Errorf("Unsupported cached value type: %v", value) + value, err := getFunc() + if err != nil { + return value, err + } + return value, conn.Put(key, value, setting.CacheService.TTLSeconds()) } } @@ -111,30 +146,34 @@ func GetInt64(key string, getFunc func() (int64, error)) (int64, error) { if conn == nil || setting.CacheService.TTL == 0 { return getFunc() } - if !conn.IsExist(key) { - var ( - value int64 - err error - ) - if value, err = getFunc(); err != nil { + + cached := conn.Get(key) + + if cached == nil { + value, err := getFunc() + if err != nil { return value, err } - err = conn.Put(key, value, setting.CacheService.TTLSeconds()) - if err != nil { - return 0, err - } + + return value, conn.Put(key, value, setting.CacheService.TTLSeconds()) } - switch value := conn.Get(key).(type) { + + switch v := conn.Get(key).(type) { case int64: - return value, nil + return v, nil case string: - v, err := strconv.ParseInt(value, 10, 64) + value, err := strconv.ParseInt(v, 10, 64) if err != nil { return 0, err } - return v, nil + return value, nil default: - return 0, fmt.Errorf("Unsupported cached value type: %v", value) + value, err := getFunc() + if err != nil { + return value, err + } + + return value, conn.Put(key, value, setting.CacheService.TTLSeconds()) } } diff --git a/modules/system/setting.go b/modules/system/setting.go deleted file mode 100644 index aebf24a501..0000000000 --- a/modules/system/setting.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package system - -import ( - "strconv" - - "code.gitea.io/gitea/models/system" - "code.gitea.io/gitea/modules/cache" -) - -func genKey(key string) string { - return "system.setting." + key -} - -// GetSetting returns the setting value via the key -func GetSetting(key string) (string, error) { - return cache.GetString(genKey(key), func() (string, error) { - res, err := system.GetSetting(key) - if err != nil { - return "", err - } - return res.SettingValue, nil - }) -} - -// GetSettingBool return bool value of setting, -// none existing keys and errors are ignored and result in false -func GetSettingBool(key string) bool { - s, _ := GetSetting(key) - b, _ := strconv.ParseBool(s) - return b -} - -// SetSetting sets the setting value -func SetSetting(key, value string, version int) error { - cache.Remove(genKey(key)) - - return system.SetSetting(&system.Setting{ - SettingKey: key, - SettingValue: value, - Version: version, - }) -} diff --git a/modules/system/user_setting.go b/modules/system/user_setting.go deleted file mode 100644 index eaf146c08d..0000000000 --- a/modules/system/user_setting.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2021 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package system - -import ( - "fmt" - - "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/cache" -) - -func genUserKey(userID int64, key string) string { - return fmt.Sprintf("user_%d.setting.%s", userID, key) -} - -// GetUserSetting returns the user setting value via the key -func GetUserSetting(userID int64, key string) (string, error) { - return cache.GetString(genUserKey(userID, key), func() (string, error) { - res, err := user.GetSetting(userID, key) - if err != nil { - return "", err - } - return res.SettingValue, nil - }) -} - -// SetUserSetting sets the user setting value -func SetUserSetting(userID int64, key, value string) error { - cache.Remove(genUserKey(userID, key)) - - return user.SetUserSetting(userID, key, value) -} diff --git a/modules/templates/helper.go b/modules/templates/helper.go index a723291440..dc90907e01 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -42,7 +42,6 @@ import ( "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/svg" - system_module "code.gitea.io/gitea/modules/system" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/gitdiff" @@ -87,7 +86,7 @@ func NewFuncMap() []template.FuncMap { return setting.AssetVersion }, "DisableGravatar": func() bool { - return system_module.GetSettingBool(system_model.KeyPictureDisableGravatar) + return system_model.GetSettingBool(system_model.KeyPictureDisableGravatar) }, "DefaultShowFullName": func() bool { return setting.UI.DefaultShowFullName diff --git a/routers/web/admin/config.go b/routers/web/admin/config.go index 614d3d4f66..792eec8d56 100644 --- a/routers/web/admin/config.go +++ b/routers/web/admin/config.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" - system_module "code.gitea.io/gitea/modules/system" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/mailer" @@ -203,7 +202,11 @@ func ChangeConfig(ctx *context.Context) { value := ctx.FormString("value") version := ctx.FormInt("version") - if err := system_module.SetSetting(key, value, version); err != nil { + if err := system_model.SetSetting(&system_model.Setting{ + SettingKey: key, + SettingValue: value, + Version: version, + }); err != nil { log.Error("set setting failed: %v", err) ctx.JSON(http.StatusOK, map[string]string{ "err": ctx.Tr("admin.config.set_setting_failed", key), From 87d05d376dd93fcdf9396a6c57eac4dc0567de1e Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Thu, 10 Nov 2022 22:22:41 +0800 Subject: [PATCH 015/182] Init git module before database migration (#21764) (#21765) Backport #21764 Some database migrations depend on the git module. --- models/migrations/migrations.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index f1f943a2c2..2e661531b6 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -14,6 +14,7 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -513,6 +514,13 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t return nil } + // Some migration tasks depend on the git command + if git.DefaultContext == nil { + if err = git.InitSimple(context.Background()); err != nil { + return err + } + } + // Migrate for i, m := range migrations[v-minDBVersion:] { log.Info("Migration[%d]: %s", v+int64(i), m.Description()) From 3aacc9b4accb69cdcd1ec6eac611e0b0819f0b9a Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Fri, 11 Nov 2022 05:11:56 +0800 Subject: [PATCH 016/182] Revert unrelated changes for SMTP auth (#21767) (#21768) Backport #21767 The purpose of #18982 is to improve the SMTP mailer, but there were some unrelated changes made to the SMTP auth in https://github.com/go-gitea/gitea/pull/18982/commits/d60c43869420f5fc43ad19b454c9ae50dad65964 This PR reverts these unrelated changes, fix #21744 Co-authored-by: Lunny Xiao --- cmd/admin.go | 8 ++++---- routers/web/admin/auths.go | 2 +- services/auth/source/smtp/auth.go | 6 +++--- services/auth/source/smtp/source.go | 2 +- services/auth/source/smtp/source_authenticate.go | 2 +- services/forms/auth_form.go | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index 525bc2cfcd..d33d17a53d 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -413,9 +413,9 @@ var ( Usage: "SMTP Authentication Type (PLAIN/LOGIN/CRAM-MD5) default PLAIN", }, cli.StringFlag{ - Name: "addr", + Name: "host", Value: "", - Usage: "SMTP Addr", + Usage: "SMTP Host", }, cli.IntFlag{ Name: "port", @@ -955,8 +955,8 @@ func parseSMTPConfig(c *cli.Context, conf *smtp.Source) error { } conf.Auth = c.String("auth-type") } - if c.IsSet("addr") { - conf.Addr = c.String("addr") + if c.IsSet("host") { + conf.Host = c.String("host") } if c.IsSet("port") { conf.Port = c.Int("port") diff --git a/routers/web/admin/auths.go b/routers/web/admin/auths.go index b79b317555..7ea8a52809 100644 --- a/routers/web/admin/auths.go +++ b/routers/web/admin/auths.go @@ -159,7 +159,7 @@ func parseLDAPConfig(form forms.AuthenticationForm) *ldap.Source { func parseSMTPConfig(form forms.AuthenticationForm) *smtp.Source { return &smtp.Source{ Auth: form.SMTPAuth, - Addr: form.SMTPAddr, + Host: form.SMTPHost, Port: form.SMTPPort, AllowedDomains: form.AllowedDomains, ForceSMTPS: form.ForceSMTPS, diff --git a/services/auth/source/smtp/auth.go b/services/auth/source/smtp/auth.go index 487c049722..e8453fde69 100644 --- a/services/auth/source/smtp/auth.go +++ b/services/auth/source/smtp/auth.go @@ -58,10 +58,10 @@ var ErrUnsupportedLoginType = errors.New("Login source is unknown") func Authenticate(a smtp.Auth, source *Source) error { tlsConfig := &tls.Config{ InsecureSkipVerify: source.SkipVerify, - ServerName: source.Addr, + ServerName: source.Host, } - conn, err := net.Dial("tcp", net.JoinHostPort(source.Addr, strconv.Itoa(source.Port))) + conn, err := net.Dial("tcp", net.JoinHostPort(source.Host, strconv.Itoa(source.Port))) if err != nil { return err } @@ -71,7 +71,7 @@ func Authenticate(a smtp.Auth, source *Source) error { conn = tls.Client(conn, tlsConfig) } - client, err := smtp.NewClient(conn, source.Addr) + client, err := smtp.NewClient(conn, source.Host) if err != nil { return fmt.Errorf("failed to create NewClient: %w", err) } diff --git a/services/auth/source/smtp/source.go b/services/auth/source/smtp/source.go index b2286d42a0..5e69f912da 100644 --- a/services/auth/source/smtp/source.go +++ b/services/auth/source/smtp/source.go @@ -19,7 +19,7 @@ import ( // Source holds configuration for the SMTP login source. type Source struct { Auth string - Addr string + Host string Port int AllowedDomains string `xorm:"TEXT"` ForceSMTPS bool diff --git a/services/auth/source/smtp/source_authenticate.go b/services/auth/source/smtp/source_authenticate.go index 63fd3e5511..dff24d494e 100644 --- a/services/auth/source/smtp/source_authenticate.go +++ b/services/auth/source/smtp/source_authenticate.go @@ -32,7 +32,7 @@ func (source *Source) Authenticate(user *user_model.User, userName, password str var auth smtp.Auth switch source.Auth { case PlainAuthentication: - auth = smtp.PlainAuth("", userName, password, source.Addr) + auth = smtp.PlainAuth("", userName, password, source.Host) case LoginAuthentication: auth = &loginAuthenticator{userName, password} case CRAMMD5Authentication: diff --git a/services/forms/auth_form.go b/services/forms/auth_form.go index 9064be2cca..7e7c756752 100644 --- a/services/forms/auth_form.go +++ b/services/forms/auth_form.go @@ -45,7 +45,7 @@ type AuthenticationForm struct { IsActive bool IsSyncEnabled bool SMTPAuth string - SMTPAddr string + SMTPHost string SMTPPort int AllowedDomains string SecurityProtocol int `binding:"Range(0,2)"` From 169eeee1018ee4f578f70a05cd135b2517a42a3f Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 11 Nov 2022 14:26:17 +0800 Subject: [PATCH 017/182] Set last login when activating account (#21731) (#21755) Backport #21731. Fix #21698. Set the last login time to the current time when activating the user successfully. Co-authored-by: Lunny Xiao Co-authored-by: Lunny Xiao --- routers/web/auth/auth.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/routers/web/auth/auth.go b/routers/web/auth/auth.go index 25d70d7c47..0f8128946c 100644 --- a/routers/web/auth/auth.go +++ b/routers/web/auth/auth.go @@ -783,6 +783,13 @@ func handleAccountActivation(ctx *context.Context, user *user_model.User) { return } + // Register last login + user.SetLastLogin() + if err := user_model.UpdateUserCols(ctx, user, "last_login_unix"); err != nil { + ctx.ServerError("UpdateUserCols", err) + return + } + ctx.Flash.Success(ctx.Tr("auth.account_activated")) ctx.Redirect(setting.AppSubURL + "/") } From e4bf9cad1ea4ddfa184a110ca0f44bee90399048 Mon Sep 17 00:00:00 2001 From: silverwind Date: Fri, 11 Nov 2022 14:14:16 +0100 Subject: [PATCH 018/182] Ignore line anchor links with leading zeroes (#21728) (#21776) Backport #21728 Fixes: https://github.com/go-gitea/gitea/issues/21722 Co-authored-by: Lunny Xiao --- web_src/js/features/repo-code.js | 37 ++++++++++++++++----------- web_src/js/features/repo-code.test.js | 18 +++++++++++++ 2 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 web_src/js/features/repo-code.test.js diff --git a/web_src/js/features/repo-code.js b/web_src/js/features/repo-code.js index 002a25f6ed..083a17bf21 100644 --- a/web_src/js/features/repo-code.js +++ b/web_src/js/features/repo-code.js @@ -4,6 +4,9 @@ import {invertFileFolding} from './file-fold.js'; import {createTippy} from '../modules/tippy.js'; import {copyToClipboard} from './clipboard.js'; +export const singleAnchorRegex = /^#(L|n)([1-9][0-9]*)$/; +export const rangeAnchorRegex = /^#(L[1-9][0-9]*)-(L[1-9][0-9]*)$/; + function changeHash(hash) { if (window.history.pushState) { window.history.pushState(null, null, hash); @@ -135,7 +138,7 @@ export function initRepoCodeView() { }); $(window).on('hashchange', () => { - let m = window.location.hash.match(/^#(L\d+)-(L\d+)$/); + let m = window.location.hash.match(rangeAnchorRegex); let $list; if ($('div.blame').length) { $list = $('.code-view td.lines-code.blame-code'); @@ -145,27 +148,31 @@ export function initRepoCodeView() { let $first; if (m) { $first = $list.filter(`[rel=${m[1]}]`); - selectRange($list, $first, $list.filter(`[rel=${m[2]}]`)); + if ($first.length) { + selectRange($list, $first, $list.filter(`[rel=${m[2]}]`)); - // show code view menu marker (don't show in blame page) - if ($('div.blame').length === 0) { - showLineButton(); + // show code view menu marker (don't show in blame page) + if ($('div.blame').length === 0) { + showLineButton(); + } + + $('html, body').scrollTop($first.offset().top - 200); + return; } - - $('html, body').scrollTop($first.offset().top - 200); - return; } - m = window.location.hash.match(/^#(L|n)(\d+)$/); + m = window.location.hash.match(singleAnchorRegex); if (m) { $first = $list.filter(`[rel=L${m[2]}]`); - selectRange($list, $first); + if ($first.length) { + selectRange($list, $first); - // show code view menu marker (don't show in blame page) - if ($('div.blame').length === 0) { - showLineButton(); + // show code view menu marker (don't show in blame page) + if ($('div.blame').length === 0) { + showLineButton(); + } + + $('html, body').scrollTop($first.offset().top - 200); } - - $('html, body').scrollTop($first.offset().top - 200); } }).trigger('hashchange'); } diff --git a/web_src/js/features/repo-code.test.js b/web_src/js/features/repo-code.test.js new file mode 100644 index 0000000000..3bd1973c1f --- /dev/null +++ b/web_src/js/features/repo-code.test.js @@ -0,0 +1,18 @@ +import {test, expect} from 'vitest'; +import {singleAnchorRegex, rangeAnchorRegex} from './repo-code.js'; + +test('singleAnchorRegex', () => { + expect(singleAnchorRegex.test('#L0')).toEqual(false); + expect(singleAnchorRegex.test('#L1')).toEqual(true); + expect(singleAnchorRegex.test('#L01')).toEqual(false); + expect(singleAnchorRegex.test('#n0')).toEqual(false); + expect(singleAnchorRegex.test('#n1')).toEqual(true); + expect(singleAnchorRegex.test('#n01')).toEqual(false); +}); + +test('rangeAnchorRegex', () => { + expect(rangeAnchorRegex.test('#L0-L10')).toEqual(false); + expect(rangeAnchorRegex.test('#L1-L10')).toEqual(true); + expect(rangeAnchorRegex.test('#L01-L10')).toEqual(false); + expect(rangeAnchorRegex.test('#L1-L01')).toEqual(false); +}); From df512f77b7b6fac3efdda265a3ab9bb95a952d69 Mon Sep 17 00:00:00 2001 From: Gusted Date: Sun, 13 Nov 2022 05:15:21 +0100 Subject: [PATCH 019/182] Upgrade golang.org/x/crypto (#21792) (#21793) - Backport #21792 - Update the crypto dependency to include https://github.com/golang/crypto/commit/6fad3dfc18918c2ac9c112e46b32473bd2e5e2f9 - Resolves #17798 --- go.mod | 8 ++++---- go.sum | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 408249880c..cf1397c468 100644 --- a/go.mod +++ b/go.mod @@ -94,11 +94,11 @@ require ( github.com/yuin/goldmark-meta v1.1.0 go.jolheiser.com/hcaptcha v0.0.4 go.jolheiser.com/pwn v0.0.3 - golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be - golang.org/x/net v0.0.0-20220927171203-f486391704dc + golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891 + golang.org/x/net v0.2.0 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 - golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec - golang.org/x/text v0.3.8 + golang.org/x/sys v0.2.0 + golang.org/x/text v0.4.0 golang.org/x/tools v0.1.12 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/ini.v1 v1.67.0 diff --git a/go.sum b/go.sum index 65841b8ec3..d6748eae34 100644 --- a/go.sum +++ b/go.sum @@ -1608,8 +1608,8 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891 h1:WhEPFM1Ck5gaKybeSWvzI7Y/cd8K9K5tJGRxXMACOBA= +golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1721,8 +1721,8 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220927171203-f486391704dc h1:FxpXZdoBqT8RjqTy6i1E8nXHhW21wK7ptQ/EPIGxzPQ= -golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1876,13 +1876,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1892,8 +1892,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ac409fcfba4071a0d6868e628338b175345be2ae Mon Sep 17 00:00:00 2001 From: Jason Song Date: Sun, 13 Nov 2022 13:54:24 +0800 Subject: [PATCH 020/182] Load GitRepo in API before deleting issue (#21720) (#21796) Backport #21720. Fix #20921. The `ctx.Repo.GitRepo` has been used in deleting issues when the issue is a PR. Co-authored-by: Lunny Xiao Co-authored-by: Lauris BH Co-authored-by: Lunny Xiao Co-authored-by: Lauris BH --- routers/api/v1/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 0d11674aa9..a515551a57 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -898,7 +898,7 @@ func Routes(ctx gocontext.Context) *web.Route { m.Group("/{index}", func() { m.Combo("").Get(repo.GetIssue). Patch(reqToken(), bind(api.EditIssueOption{}), repo.EditIssue). - Delete(reqToken(), reqAdmin(), repo.DeleteIssue) + Delete(reqToken(), reqAdmin(), context.ReferencesGitRepo(), repo.DeleteIssue) m.Group("/comments", func() { m.Combo("").Get(repo.ListIssueComments). Post(reqToken(), mustNotBeArchived, bind(api.CreateIssueCommentOption{}), repo.CreateIssueComment) From 0d25292fbc05a71d21a458cec780e795d294e63e Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 14 Nov 2022 02:58:32 +0000 Subject: [PATCH 021/182] Prevent panic in doctor command when running default checks (#21791) (#21807) Backport #21791 There was a bug introduced in #21352 due to a change of behaviour caused by #19280. This causes a panic on running the default doctor checks because the panic introduced by #19280 assumes that the only way opts.StdOut and opts.Stderr can be set in RunOpts is deliberately. Unfortunately, when running a git.Command the provided RunOpts can be set, therefore if you share a common set of RunOpts these two values can be set by the previous commands. This PR stops using common RunOpts for the commands in that doctor check but secondly stops RunCommand variants from changing the provided RunOpts. Signed-off-by: Andrew Thornton --- modules/doctor/heads.go | 10 ++++------ modules/git/command.go | 26 ++++++++++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/modules/doctor/heads.go b/modules/doctor/heads.go index 7f3b2a8a02..b1bfd50b20 100644 --- a/modules/doctor/heads.go +++ b/modules/doctor/heads.go @@ -19,11 +19,9 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool) numReposUpdated := 0 err := iterateRepositories(ctx, func(repo *repo_model.Repository) error { numRepos++ - runOpts := &git.RunOpts{Dir: repo.RepoPath()} + _, _, defaultBranchErr := git.NewCommand(ctx, "rev-parse").AddDashesAndList(repo.DefaultBranch).RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) - _, _, defaultBranchErr := git.NewCommand(ctx, "rev-parse").AddDashesAndList(repo.DefaultBranch).RunStdString(runOpts) - - head, _, headErr := git.NewCommand(ctx, "symbolic-ref", "--short", "HEAD").RunStdString(runOpts) + head, _, headErr := git.NewCommand(ctx, "symbolic-ref", "--short", "HEAD").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // what we expect: default branch is valid, and HEAD points to it if headErr == nil && defaultBranchErr == nil && head == repo.DefaultBranch { @@ -49,7 +47,7 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool) } // otherwise, let's try fixing HEAD - err := git.NewCommand(ctx, "symbolic-ref").AddDashesAndList("HEAD", repo.DefaultBranch).Run(runOpts) + err := git.NewCommand(ctx, "symbolic-ref").AddDashesAndList("HEAD", git.BranchPrefix+repo.DefaultBranch).Run(&git.RunOpts{Dir: repo.RepoPath()}) if err != nil { logger.Warn("Failed to fix HEAD for %s/%s: %v", repo.OwnerName, repo.Name, err) return nil @@ -65,7 +63,7 @@ func synchronizeRepoHeads(ctx context.Context, logger log.Logger, autofix bool) logger.Info("Out of %d repos, HEADs for %d are now fixed and HEADS for %d are still broken", numRepos, numReposUpdated, numDefaultBranchesBroken+numHeadsBroken-numReposUpdated) } else { if numHeadsBroken == 0 && numDefaultBranchesBroken == 0 { - logger.Info("All %d repos have their HEADs in the correct state") + logger.Info("All %d repos have their HEADs in the correct state", numRepos) } else { if numHeadsBroken == 0 && numDefaultBranchesBroken != 0 { logger.Critical("Default branches are broken for %d/%d repos", numDefaultBranchesBroken, numRepos) diff --git a/modules/git/command.go b/modules/git/command.go index abf40b0cd7..0d94494f11 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -202,8 +202,11 @@ func (c *Command) Run(opts *RunOpts) error { if opts == nil { opts = &RunOpts{} } - if opts.Timeout <= 0 { - opts.Timeout = defaultCommandExecutionTimeout + + // We must not change the provided options + timeout := opts.Timeout + if timeout <= 0 { + timeout = defaultCommandExecutionTimeout } if len(opts.Dir) == 0 { @@ -238,7 +241,7 @@ func (c *Command) Run(opts *RunOpts) error { if opts.UseContextTimeout { ctx, cancel, finished = process.GetManager().AddContext(c.parentContext, desc) } else { - ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, opts.Timeout, desc) + ctx, cancel, finished = process.GetManager().AddContextTimeout(c.parentContext, timeout, desc) } defer finished() @@ -339,9 +342,20 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS } stdoutBuf := &bytes.Buffer{} stderrBuf := &bytes.Buffer{} - opts.Stdout = stdoutBuf - opts.Stderr = stderrBuf - err := c.Run(opts) + + // We must not change the provided options as it could break future calls - therefore make a copy. + newOpts := &RunOpts{ + Env: opts.Env, + Timeout: opts.Timeout, + UseContextTimeout: opts.UseContextTimeout, + Dir: opts.Dir, + Stdout: stdoutBuf, + Stderr: stderrBuf, + Stdin: opts.Stdin, + PipelineFunc: opts.PipelineFunc, + } + + err := c.Run(newOpts) stderr = stderrBuf.Bytes() if err != nil { return nil, stderr, &runStdError{err: err, stderr: bytesToString(stderr)} From 8917af870139dbc0b4b7a2be55613f2c87b38996 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 17 Nov 2022 03:48:33 +0800 Subject: [PATCH 022/182] Ignore issue template with a special name (#21830) (#21835) Backport #21830. A file in `ISSUE_TEMPLATE` with the name `config.yml` shouldn't be treated as a YAML template, it's for [configuring the template chooser](https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/configuring-issue-templates-for-your-repository#configuring-the-template-chooser). The old code tried to ignore the file, but it didn't work, caused by #20987. That's why the warning is displayed: image Note that this PR is not an implementation of `config.yml`, there will be another one to do it. --- modules/structs/issue.go | 6 ++--- modules/structs/issue_test.go | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 modules/structs/issue_test.go diff --git a/modules/structs/issue.go b/modules/structs/issue.go index 70f5e1ba8e..25c6251fbf 100644 --- a/modules/structs/issue.go +++ b/modules/structs/issue.go @@ -5,7 +5,7 @@ package structs import ( - "path/filepath" + "path" "time" ) @@ -163,11 +163,11 @@ const ( // Type returns the type of IssueTemplate, can be "md", "yaml" or empty for known func (it IssueTemplate) Type() IssueTemplateType { - if it.Name == "config.yaml" || it.Name == "config.yml" { + if base := path.Base(it.FileName); base == "config.yaml" || base == "config.yml" { // ignore config.yaml which is a special configuration file return "" } - if ext := filepath.Ext(it.FileName); ext == ".md" { + if ext := path.Ext(it.FileName); ext == ".md" { return IssueTemplateTypeMarkdown } else if ext == ".yaml" || ext == ".yml" { return IssueTemplateTypeYaml diff --git a/modules/structs/issue_test.go b/modules/structs/issue_test.go new file mode 100644 index 0000000000..5312585d0f --- /dev/null +++ b/modules/structs/issue_test.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package structs + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIssueTemplate_Type(t *testing.T) { + tests := []struct { + fileName string + want IssueTemplateType + }{ + { + fileName: ".gitea/ISSUE_TEMPLATE/bug_report.yaml", + want: IssueTemplateTypeYaml, + }, + { + fileName: ".gitea/ISSUE_TEMPLATE/bug_report.md", + want: IssueTemplateTypeMarkdown, + }, + { + fileName: ".gitea/ISSUE_TEMPLATE/bug_report.txt", + want: "", + }, + { + fileName: ".gitea/ISSUE_TEMPLATE/config.yaml", + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.fileName, func(t *testing.T) { + it := IssueTemplate{ + FileName: tt.fileName, + } + assert.Equal(t, tt.want, it.Type()) + }) + } +} From af8b2250c449dbbd9df3d40c2c8847d39d43c853 Mon Sep 17 00:00:00 2001 From: Gusted Date: Fri, 18 Nov 2022 15:25:00 +0100 Subject: [PATCH 023/182] Prevent dangling user redirects (#21856) (#21858) - Backport #21856 - It's possible that the `user_redirect` table contains a user id that no longer exists. - Delete a user redirect upon deleting the user. - Add a check for these dangling user redirects to check-db-consistency. --- models/user.go | 1 + modules/doctor/dbconsistency.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/models/user.go b/models/user.go index 85f465127a..0fc28ff055 100644 --- a/models/user.go +++ b/models/user.go @@ -89,6 +89,7 @@ func DeleteUser(ctx context.Context, u *user_model.User, purge bool) (err error) &user_model.UserBadge{UserID: u.ID}, &pull_model.AutoMerge{DoerID: u.ID}, &pull_model.ReviewState{UserID: u.ID}, + &user_model.Redirect{RedirectUserID: u.ID}, ); err != nil { return fmt.Errorf("deleteBeans: %w", err) } diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go index 7ae349908e..89d974a350 100644 --- a/modules/doctor/dbconsistency.go +++ b/modules/doctor/dbconsistency.go @@ -205,6 +205,9 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er // find stopwatches without existing issue genericOrphanCheck("Orphaned Stopwatches without existing Issue", "stopwatch", "issue", "stopwatch.issue_id=`issue`.id"), + // find redirects without existing user. + genericOrphanCheck("Orphaned Redirects without existing redirect user", + "user_redirect", "user", "user_redirect.redirect_user_id=`user`.id"), ) for _, c := range consistencyChecks { From 7a004ad7eb02b4295e78142af10ae18cf2ed9827 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Sun, 20 Nov 2022 18:44:20 +0800 Subject: [PATCH 024/182] Support comma-delimited string as labels in issue template (#21831) (#21873) Backport #21831. The [labels in issue YAML templates](https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms#top-level-syntax) can be a string array or a comma-delimited string, so a single string should be valid labels. The old codes committed in #20987 ignore this, that's why the warning is displayed: image Fixes #17877. --- modules/issue/template/template.go | 4 +- modules/issue/template/template_test.go | 326 ++++++++++++++++-------- modules/issue/template/unmarshal.go | 2 +- modules/markup/markdown/meta_test.go | 48 ++-- modules/structs/issue.go | 53 +++- modules/structs/issue_test.go | 63 +++++ templates/swagger/v1_json.tmpl | 13 +- 7 files changed, 369 insertions(+), 140 deletions(-) diff --git a/modules/issue/template/template.go b/modules/issue/template/template.go index 3b33852cb5..0bdf5a1987 100644 --- a/modules/issue/template/template.go +++ b/modules/issue/template/template.go @@ -165,7 +165,7 @@ func validateOptions(field *api.IssueFormField, idx int) error { return position.Errorf("should be a string") } case api.IssueFormFieldTypeCheckboxes: - opt, ok := option.(map[interface{}]interface{}) + opt, ok := option.(map[string]interface{}) if !ok { return position.Errorf("should be a dictionary") } @@ -351,7 +351,7 @@ func (o *valuedOption) Label() string { return label } case api.IssueFormFieldTypeCheckboxes: - if vs, ok := o.data.(map[interface{}]interface{}); ok { + if vs, ok := o.data.(map[string]interface{}); ok { if v, ok := vs["label"].(string); ok { return v } diff --git a/modules/issue/template/template_test.go b/modules/issue/template/template_test.go index 883e1e0780..c3863a64a6 100644 --- a/modules/issue/template/template_test.go +++ b/modules/issue/template/template_test.go @@ -6,18 +6,21 @@ package template import ( "net/url" - "reflect" "testing" "code.gitea.io/gitea/modules/json" api "code.gitea.io/gitea/modules/structs" + + "github.com/stretchr/testify/require" ) func TestValidate(t *testing.T) { tests := []struct { - name string - content string - wantErr string + name string + filename string + content string + want *api.IssueTemplate + wantErr string }{ { name: "miss name", @@ -316,21 +319,9 @@ body: `, wantErr: "body[0](checkboxes), option[0]: 'required' should be a bool", }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tmpl, err := unmarshal("test.yaml", []byte(tt.content)) - if err != nil { - t.Fatal(err) - } - if err := Validate(tmpl); (err == nil) != (tt.wantErr == "") || err != nil && err.Error() != tt.wantErr { - t.Errorf("Validate() error = %v, wantErr %q", err, tt.wantErr) - } - }) - } - - t.Run("valid", func(t *testing.T) { - content := ` + { + name: "valid", + content: ` name: Name title: Title about: About @@ -386,96 +377,227 @@ body: required: false - label: Option 3 of checkboxes required: true -` - want := &api.IssueTemplate{ - Name: "Name", - Title: "Title", - About: "About", - Labels: []string{"label1", "label2"}, - Ref: "Ref", - Fields: []*api.IssueFormField{ - { - Type: "markdown", - ID: "id1", - Attributes: map[string]interface{}{ - "value": "Value of the markdown", - }, - }, - { - Type: "textarea", - ID: "id2", - Attributes: map[string]interface{}{ - "label": "Label of textarea", - "description": "Description of textarea", - "placeholder": "Placeholder of textarea", - "value": "Value of textarea", - "render": "bash", - }, - Validations: map[string]interface{}{ - "required": true, - }, - }, - { - Type: "input", - ID: "id3", - Attributes: map[string]interface{}{ - "label": "Label of input", - "description": "Description of input", - "placeholder": "Placeholder of input", - "value": "Value of input", - }, - Validations: map[string]interface{}{ - "required": true, - "is_number": true, - "regex": "[a-zA-Z0-9]+", - }, - }, - { - Type: "dropdown", - ID: "id4", - Attributes: map[string]interface{}{ - "label": "Label of dropdown", - "description": "Description of dropdown", - "multiple": true, - "options": []interface{}{ - "Option 1 of dropdown", - "Option 2 of dropdown", - "Option 3 of dropdown", +`, + want: &api.IssueTemplate{ + Name: "Name", + Title: "Title", + About: "About", + Labels: []string{"label1", "label2"}, + Ref: "Ref", + Fields: []*api.IssueFormField{ + { + Type: "markdown", + ID: "id1", + Attributes: map[string]interface{}{ + "value": "Value of the markdown", }, }, - Validations: map[string]interface{}{ - "required": true, + { + Type: "textarea", + ID: "id2", + Attributes: map[string]interface{}{ + "label": "Label of textarea", + "description": "Description of textarea", + "placeholder": "Placeholder of textarea", + "value": "Value of textarea", + "render": "bash", + }, + Validations: map[string]interface{}{ + "required": true, + }, }, - }, - { - Type: "checkboxes", - ID: "id5", - Attributes: map[string]interface{}{ - "label": "Label of checkboxes", - "description": "Description of checkboxes", - "options": []interface{}{ - map[interface{}]interface{}{"label": "Option 1 of checkboxes", "required": true}, - map[interface{}]interface{}{"label": "Option 2 of checkboxes", "required": false}, - map[interface{}]interface{}{"label": "Option 3 of checkboxes", "required": true}, + { + Type: "input", + ID: "id3", + Attributes: map[string]interface{}{ + "label": "Label of input", + "description": "Description of input", + "placeholder": "Placeholder of input", + "value": "Value of input", + }, + Validations: map[string]interface{}{ + "required": true, + "is_number": true, + "regex": "[a-zA-Z0-9]+", + }, + }, + { + Type: "dropdown", + ID: "id4", + Attributes: map[string]interface{}{ + "label": "Label of dropdown", + "description": "Description of dropdown", + "multiple": true, + "options": []interface{}{ + "Option 1 of dropdown", + "Option 2 of dropdown", + "Option 3 of dropdown", + }, + }, + Validations: map[string]interface{}{ + "required": true, + }, + }, + { + Type: "checkboxes", + ID: "id5", + Attributes: map[string]interface{}{ + "label": "Label of checkboxes", + "description": "Description of checkboxes", + "options": []interface{}{ + map[string]interface{}{"label": "Option 1 of checkboxes", "required": true}, + map[string]interface{}{"label": "Option 2 of checkboxes", "required": false}, + map[string]interface{}{"label": "Option 3 of checkboxes", "required": true}, + }, }, }, }, + FileName: "test.yaml", }, - FileName: "test.yaml", - } - got, err := unmarshal("test.yaml", []byte(content)) - if err != nil { - t.Fatal(err) - } - if err := Validate(got); err != nil { - t.Errorf("Validate() error = %v", err) - } - if !reflect.DeepEqual(want, got) { - jsonWant, _ := json.Marshal(want) - jsonGot, _ := json.Marshal(got) - t.Errorf("want:\n%s\ngot:\n%s", jsonWant, jsonGot) - } - }) + wantErr: "", + }, + { + name: "single label", + content: ` +name: Name +title: Title +about: About +labels: label1 +ref: Ref +body: + - type: markdown + id: id1 + attributes: + value: Value of the markdown +`, + want: &api.IssueTemplate{ + Name: "Name", + Title: "Title", + About: "About", + Labels: []string{"label1"}, + Ref: "Ref", + Fields: []*api.IssueFormField{ + { + Type: "markdown", + ID: "id1", + Attributes: map[string]interface{}{ + "value": "Value of the markdown", + }, + }, + }, + FileName: "test.yaml", + }, + wantErr: "", + }, + { + name: "comma-delimited labels", + content: ` +name: Name +title: Title +about: About +labels: label1,label2,,label3 ,, +ref: Ref +body: + - type: markdown + id: id1 + attributes: + value: Value of the markdown +`, + want: &api.IssueTemplate{ + Name: "Name", + Title: "Title", + About: "About", + Labels: []string{"label1", "label2", "label3"}, + Ref: "Ref", + Fields: []*api.IssueFormField{ + { + Type: "markdown", + ID: "id1", + Attributes: map[string]interface{}{ + "value": "Value of the markdown", + }, + }, + }, + FileName: "test.yaml", + }, + wantErr: "", + }, + { + name: "empty string as labels", + content: ` +name: Name +title: Title +about: About +labels: '' +ref: Ref +body: + - type: markdown + id: id1 + attributes: + value: Value of the markdown +`, + want: &api.IssueTemplate{ + Name: "Name", + Title: "Title", + About: "About", + Labels: nil, + Ref: "Ref", + Fields: []*api.IssueFormField{ + { + Type: "markdown", + ID: "id1", + Attributes: map[string]interface{}{ + "value": "Value of the markdown", + }, + }, + }, + FileName: "test.yaml", + }, + wantErr: "", + }, + { + name: "comma delimited labels in markdown", + filename: "test.md", + content: `--- +name: Name +title: Title +about: About +labels: label1,label2,,label3 ,, +ref: Ref +--- +Content +`, + want: &api.IssueTemplate{ + Name: "Name", + Title: "Title", + About: "About", + Labels: []string{"label1", "label2", "label3"}, + Ref: "Ref", + Fields: nil, + Content: "Content\n", + FileName: "test.md", + }, + wantErr: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + filename := "test.yaml" + if tt.filename != "" { + filename = tt.filename + } + tmpl, err := unmarshal(filename, []byte(tt.content)) + require.NoError(t, err) + if tt.wantErr != "" { + require.EqualError(t, Validate(tmpl), tt.wantErr) + } else { + require.NoError(t, Validate(tmpl)) + want, _ := json.Marshal(tt.want) + got, _ := json.Marshal(tmpl) + require.JSONEq(t, string(want), string(got)) + } + }) + } } func TestRenderToMarkdown(t *testing.T) { diff --git a/modules/issue/template/unmarshal.go b/modules/issue/template/unmarshal.go index 24587b0fed..3398719cf6 100644 --- a/modules/issue/template/unmarshal.go +++ b/modules/issue/template/unmarshal.go @@ -16,7 +16,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // CouldBe indicates a file with the filename could be a template, diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go index 720d0066f4..1e9768e618 100644 --- a/modules/markup/markdown/meta_test.go +++ b/modules/markup/markdown/meta_test.go @@ -9,82 +9,86 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/structs" - "github.com/stretchr/testify/assert" ) -func validateMetadata(it structs.IssueTemplate) bool { - /* - A legacy to keep the unit tests working. - Copied from the method "func (it IssueTemplate) Valid() bool", the original method has been removed. - Because it becomes quite complicated to validate an issue template which is support yaml form now. - The new way to validate an issue template is to call the Validate in modules/issue/template, - */ +/* +IssueTemplate is a legacy to keep the unit tests working. +Copied from structs.IssueTemplate, the original type has been changed a lot to support yaml template. +*/ +type IssueTemplate struct { + Name string `json:"name" yaml:"name"` + Title string `json:"title" yaml:"title"` + About string `json:"about" yaml:"about"` + Labels []string `json:"labels" yaml:"labels"` + Ref string `json:"ref" yaml:"ref"` +} + +func (it *IssueTemplate) Valid() bool { return strings.TrimSpace(it.Name) != "" && strings.TrimSpace(it.About) != "" } func TestExtractMetadata(t *testing.T) { t.Run("ValidFrontAndBody", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest), &meta) assert.NoError(t, err) assert.Equal(t, bodyTest, body) assert.Equal(t, metaTest, meta) - assert.True(t, validateMetadata(meta)) + assert.True(t, meta.Valid()) }) t.Run("NoFirstSeparator", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate _, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoLastSeparator", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate _, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest), &meta) assert.Error(t, err) }) t.Run("NoBody", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate body, err := ExtractMetadata(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest), &meta) assert.NoError(t, err) assert.Equal(t, "", body) assert.Equal(t, metaTest, meta) - assert.True(t, validateMetadata(meta)) + assert.True(t, meta.Valid()) }) } func TestExtractMetadataBytes(t *testing.T) { t.Run("ValidFrontAndBody", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) assert.NoError(t, err) assert.Equal(t, bodyTest, string(body)) assert.Equal(t, metaTest, meta) - assert.True(t, validateMetadata(meta)) + assert.True(t, meta.Valid()) }) t.Run("NoFirstSeparator", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", frontTest, sepTest, bodyTest)), &meta) assert.Error(t, err) }) t.Run("NoLastSeparator", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate _, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, bodyTest)), &meta) assert.Error(t, err) }) t.Run("NoBody", func(t *testing.T) { - var meta structs.IssueTemplate + var meta IssueTemplate body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) assert.NoError(t, err) assert.Equal(t, "", string(body)) assert.Equal(t, metaTest, meta) - assert.True(t, validateMetadata(meta)) + assert.True(t, meta.Valid()) }) } @@ -97,7 +101,7 @@ labels: - bug - "test label"` bodyTest = "This is the body" - metaTest = structs.IssueTemplate{ + metaTest = IssueTemplate{ Name: "Test", About: "A Test", Title: "Test Title", diff --git a/modules/structs/issue.go b/modules/structs/issue.go index 25c6251fbf..45c3f6294a 100644 --- a/modules/structs/issue.go +++ b/modules/structs/issue.go @@ -5,8 +5,12 @@ package structs import ( + "fmt" "path" + "strings" "time" + + "gopkg.in/yaml.v3" ) // StateType issue state type @@ -143,14 +147,47 @@ type IssueFormField struct { // IssueTemplate represents an issue template for a repository // swagger:model type IssueTemplate struct { - Name string `json:"name" yaml:"name"` - Title string `json:"title" yaml:"title"` - About string `json:"about" yaml:"about"` // Using "description" in a template file is compatible - Labels []string `json:"labels" yaml:"labels"` - Ref string `json:"ref" yaml:"ref"` - Content string `json:"content" yaml:"-"` - Fields []*IssueFormField `json:"body" yaml:"body"` - FileName string `json:"file_name" yaml:"-"` + Name string `json:"name" yaml:"name"` + Title string `json:"title" yaml:"title"` + About string `json:"about" yaml:"about"` // Using "description" in a template file is compatible + Labels IssueTemplateLabels `json:"labels" yaml:"labels"` + Ref string `json:"ref" yaml:"ref"` + Content string `json:"content" yaml:"-"` + Fields []*IssueFormField `json:"body" yaml:"body"` + FileName string `json:"file_name" yaml:"-"` +} + +type IssueTemplateLabels []string + +func (l *IssueTemplateLabels) UnmarshalYAML(value *yaml.Node) error { + var labels []string + if value.IsZero() { + *l = labels + return nil + } + switch value.Kind { + case yaml.ScalarNode: + str := "" + err := value.Decode(&str) + if err != nil { + return err + } + for _, v := range strings.Split(str, ",") { + if v = strings.TrimSpace(v); v == "" { + continue + } + labels = append(labels, v) + } + *l = labels + return nil + case yaml.SequenceNode: + if err := value.Decode(&labels); err != nil { + return err + } + *l = labels + return nil + } + return fmt.Errorf("line %d: cannot unmarshal %s into IssueTemplateLabels", value.Line, value.ShortTag()) } // IssueTemplateType defines issue template type diff --git a/modules/structs/issue_test.go b/modules/structs/issue_test.go index 5312585d0f..72b40f7cf2 100644 --- a/modules/structs/issue_test.go +++ b/modules/structs/issue_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestIssueTemplate_Type(t *testing.T) { @@ -41,3 +42,65 @@ func TestIssueTemplate_Type(t *testing.T) { }) } } + +func TestIssueTemplateLabels_UnmarshalYAML(t *testing.T) { + tests := []struct { + name string + content string + tmpl *IssueTemplate + want *IssueTemplate + wantErr string + }{ + { + name: "array", + content: `labels: ["a", "b", "c"]`, + tmpl: &IssueTemplate{ + Labels: []string{"should_be_overwrote"}, + }, + want: &IssueTemplate{ + Labels: []string{"a", "b", "c"}, + }, + }, + { + name: "string", + content: `labels: "a,b,c"`, + tmpl: &IssueTemplate{ + Labels: []string{"should_be_overwrote"}, + }, + want: &IssueTemplate{ + Labels: []string{"a", "b", "c"}, + }, + }, + { + name: "empty", + content: `labels:`, + tmpl: &IssueTemplate{ + Labels: []string{"should_be_overwrote"}, + }, + want: &IssueTemplate{ + Labels: nil, + }, + }, + { + name: "error", + content: ` +labels: + a: aa + b: bb +`, + tmpl: &IssueTemplate{}, + wantErr: "line 3: cannot unmarshal !!map into IssueTemplateLabels", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := yaml.Unmarshal([]byte(tt.content), tt.tmpl) + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, tt.tmpl) + } + }) + } +} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 94fb67ab44..34dbad6dc4 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -16806,11 +16806,7 @@ "x-go-name": "FileName" }, "labels": { - "type": "array", - "items": { - "type": "string" - }, - "x-go-name": "Labels" + "$ref": "#/definitions/IssueTemplateLabels" }, "name": { "type": "string", @@ -16827,6 +16823,13 @@ }, "x-go-package": "code.gitea.io/gitea/modules/structs" }, + "IssueTemplateLabels": { + "type": "array", + "items": { + "type": "string" + }, + "x-go-package": "code.gitea.io/gitea/modules/structs" + }, "Label": { "description": "Label a label to an issue or a pr", "type": "object", From ef08998bf6711f696524c7ed5fadc8949458a8ed Mon Sep 17 00:00:00 2001 From: silverwind Date: Sun, 20 Nov 2022 11:47:02 +0100 Subject: [PATCH 025/182] Color and Style enhancements (#21784, #21799) (#21868) Backport #21784 Backport #21799 These PRs provide tweaks and simplification to the less/css selectors, simplifying text color selectors and tweak arc-green colors with a follow-up to adjust the timeline See the original PRs for more details --- templates/explore/repo_list.tmpl | 6 +- templates/org/home.tmpl | 4 +- templates/repo/diff/comments.tmpl | 2 +- templates/repo/issue/view_content.tmpl | 4 +- .../repo/issue/view_content/comments.tmpl | 20 ++-- .../repo/issue/view_content/sidebar.tmpl | 2 +- templates/shared/issuelist.tmpl | 2 +- templates/shared/user/authorlink.tmpl | 4 +- templates/shared/user/avatarlink.tmpl | 4 +- templates/shared/user/namelink.tmpl | 4 +- web_src/less/_base.less | 91 +++++-------------- web_src/less/_repository.less | 18 ++-- web_src/less/shared/issuelist.less | 8 ++ web_src/less/themes/theme-arc-green.less | 45 ++++----- 14 files changed, 81 insertions(+), 133 deletions(-) diff --git a/templates/explore/repo_list.tmpl b/templates/explore/repo_list.tmpl index 2a5c444dce..044af9522e 100644 --- a/templates/explore/repo_list.tmpl +++ b/templates/explore/repo_list.tmpl @@ -38,10 +38,10 @@ {{end}} -
+
{{if .PrimaryLanguage}} - - {{.PrimaryLanguage.Language}} + + {{.PrimaryLanguage.Language}} {{end}} {{if not $.DisableStars}} diff --git a/templates/org/home.tmpl b/templates/org/home.tmpl index 3ff86259d5..448639975f 100644 --- a/templates/org/home.tmpl +++ b/templates/org/home.tmpl @@ -68,8 +68,8 @@
{{.Name}}

- {{.NumMembers}} {{$.locale.Tr "org.lower_members"}} · - {{.NumRepos}} {{$.locale.Tr "org.lower_repositories"}} + {{.NumMembers}} {{$.locale.Tr "org.lower_members"}} · + {{.NumRepos}} {{$.locale.Tr "org.lower_repositories"}}

{{end}} diff --git a/templates/repo/diff/comments.tmpl b/templates/repo/diff/comments.tmpl index 5d298f820c..647c662d51 100644 --- a/templates/repo/diff/comments.tmpl +++ b/templates/repo/diff/comments.tmpl @@ -11,7 +11,7 @@
{{if .OriginalAuthor}} - + {{svg (MigrationIcon $.root.Repository.GetOriginalURLHostname)}} {{.OriginalAuthor}} diff --git a/templates/repo/issue/view_content.tmpl b/templates/repo/issue/view_content.tmpl index de11846af7..0cb74e8182 100644 --- a/templates/repo/issue/view_content.tmpl +++ b/templates/repo/issue/view_content.tmpl @@ -30,7 +30,7 @@
{{if .Issue.OriginalAuthor}} - + {{svg (MigrationIcon .Repository.GetOriginalURLHostname)}} {{.Issue.OriginalAuthor}} @@ -45,7 +45,7 @@ {{avatar .Issue.Poster}} - {{.Issue.Poster.GetDisplayName}} + {{template "shared/user/authorlink" .Issue.Poster}} {{.locale.Tr "repo.issues.commented_at" (.Issue.HashTag|Escape) $createdStr | Safe}} {{end}} diff --git a/templates/repo/issue/view_content/comments.tmpl b/templates/repo/issue/view_content/comments.tmpl index 7ed8cf3bde..e9b7172d1b 100644 --- a/templates/repo/issue/view_content/comments.tmpl +++ b/templates/repo/issue/view_content/comments.tmpl @@ -25,7 +25,7 @@
{{if .OriginalAuthor}} - + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} {{.OriginalAuthor}} @@ -42,9 +42,7 @@ {{end}} - - {{.Poster.GetDisplayName}} - + {{template "shared/user/authorlink" .Poster}} {{$.locale.Tr "repo.issues.commented_at" (.HashTag|Escape) $createdStr | Safe}} {{end}} @@ -151,14 +149,14 @@ {{svg "octicon-bookmark"}} {{template "shared/user/avatarlink" .Poster}} {{if eq .RefAction 3}}{{end}} - + {{template "shared/user/authorlink" .Poster}} {{$.locale.Tr $refTr (.EventTag|Escape) $createdStr (.RefCommentHTMLURL|Escape) $refFrom | Safe}} {{if eq .RefAction 3}}{{end}}
- {{.RefIssueTitle}} {{.RefIssueIdent}} + {{.RefIssueTitle}} {{.RefIssueIdent}}
{{else if eq .Type 4}} @@ -207,7 +205,7 @@ {{if .RemovedAssignee}} {{template "shared/user/avatarlink" .Assignee}} - {{.Assignee.GetDisplayName}} + {{template "shared/user/authorlink" .Assignee}} {{if eq .Poster.ID .Assignee.ID}} {{$.locale.Tr "repo.issues.remove_self_assignment" $createdStr | Safe}} {{else}} @@ -331,7 +329,7 @@
{{svg "octicon-plus"}} - + {{if eq .DependentIssue.RepoID .Issue.RepoID}} #{{.DependentIssue.Index}} {{.DependentIssue.Title}} {{else}} @@ -354,7 +352,7 @@
{{svg "octicon-trash"}} - + {{if eq .DependentIssue.RepoID .Issue.RepoID}} #{{.DependentIssue.Index}} {{.DependentIssue.Title}} {{else}} @@ -408,7 +406,7 @@
{{if .OriginalAuthor}} - + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} {{.OriginalAuthor}} @@ -536,7 +534,7 @@ {{end}} {{if .OriginalAuthor}} - + {{svg (MigrationIcon $.Repository.GetOriginalURLHostname)}} {{.OriginalAuthor}} diff --git a/templates/repo/issue/view_content/sidebar.tmpl b/templates/repo/issue/view_content/sidebar.tmpl index eb95674b13..af648387db 100644 --- a/templates/repo/issue/view_content/sidebar.tmpl +++ b/templates/repo/issue/view_content/sidebar.tmpl @@ -389,7 +389,7 @@ {{avatar $user}}
- {{$user.DisplayName}} + {{template "shared/user/authorlink" $user}}
{{$trackedtime}}
diff --git a/templates/shared/issuelist.tmpl b/templates/shared/issuelist.tmpl index 6fce412ffa..e8ad8406cd 100644 --- a/templates/shared/issuelist.tmpl +++ b/templates/shared/issuelist.tmpl @@ -160,7 +160,7 @@
{{if .NumComments}} - + {{svg "octicon-comment" 16 "mr-2"}}{{.NumComments}} {{end}} diff --git a/templates/shared/user/authorlink.tmpl b/templates/shared/user/authorlink.tmpl index 81c2a4ed5e..4a6672181f 100644 --- a/templates/shared/user/authorlink.tmpl +++ b/templates/shared/user/authorlink.tmpl @@ -1,3 +1 @@ - - {{.GetDisplayName}} - +{{.GetDisplayName}} diff --git a/templates/shared/user/avatarlink.tmpl b/templates/shared/user/avatarlink.tmpl index 40f5ee7129..90f5d96700 100644 --- a/templates/shared/user/avatarlink.tmpl +++ b/templates/shared/user/avatarlink.tmpl @@ -1,3 +1 @@ - - {{avatar .}} - +{{avatar .}} diff --git a/templates/shared/user/namelink.tmpl b/templates/shared/user/namelink.tmpl index 25dff36a78..a122f4f45b 100644 --- a/templates/shared/user/namelink.tmpl +++ b/templates/shared/user/namelink.tmpl @@ -1,3 +1 @@ - - {{.GetDisplayName}} - +{{.GetDisplayName}} diff --git a/web_src/less/_base.less b/web_src/less/_base.less index f1691d8156..09ed503776 100644 --- a/web_src/less/_base.less +++ b/web_src/less/_base.less @@ -79,6 +79,7 @@ --color-pink: #e03997; --color-brown: #a5673f; --color-grey: #888888; + --color-black: #1b1c1d; /* light variants - produced via Sass scale-color(color, $lightness: +25%) */ --color-red-light: #e45e5e; --color-orange-light: #f59555; @@ -92,9 +93,9 @@ --color-pink-light: #e86bb1; --color-brown-light: #c58b66; --color-grey-light: #a6a6a6; + --color-black-light: #525558; /* other colors */ --color-gold: #a1882b; - --color-black: #1b1c1d; --color-white: #ffffff; --color-diff-removed-word-bg: #fdb8c0; --color-diff-added-word-bg: #acf2bd; @@ -293,13 +294,15 @@ a, text-decoration-skip-ink: all; } -a.muted { +a.muted, +.muted-links a { color: inherit; } a:hover, a.muted:hover, a.muted:hover [class*="color-text"], +.muted-links a:hover, .ui.breadcrumb a:hover { color: var(--color-primary); } @@ -1301,6 +1304,22 @@ a.ui.card:hover, visibility: hidden; } +.text.red { color: var(--color-red) !important; } +.text.orange { color: var(--color-orange) !important; } +.text.yellow { color: var(--color-yellow) !important; } +.text.olive { color: var(--color-olive) !important; } +.text.green { color: var(--color-green) !important; } +.text.teal { color: var(--color-teal) !important; } +.text.blue { color: var(--color-blue) !important; } +.text.violet { color: var(--color-violet) !important; } +.text.purple { color: var(--color-purple) !important; } +.text.pink { color: var(--color-pink) !important; } +.text.brown { color: var(--color-brown) !important; } +.text.black { color: var(--color-text) !important; } +.text.grey { color: var(--color-text-light) !important; } +.text.light.grey { color: var(--color-grey-light) !important; } +.text.gold { color: var(--color-gold) !important; } + .ui { &.left:not(.action) { float: left; @@ -1370,74 +1389,6 @@ a.ui.card:hover, } .text { - &.red { - color: var(--color-red) !important; - - a { - color: inherit !important; - - &:hover { - color: var(--color-red-light) !important; - } - } - } - - &.blue { - color: var(--color-blue) !important; - - a { - color: inherit !important; - - &:hover { - color: var(--color-blue-light) !important; - } - } - } - - &.black { - color: var(--color-text); - - &:hover { - color: var(--color-text-dark); - } - } - - &.grey { - color: var(--color-text-light) !important; - - a { - color: var(--color-text) !important; - - &:hover { - color: var(--color-primary) !important; - } - } - } - - &.light.grey { - color: var(--color-text-light-2) !important; - } - - &.green { - color: var(--color-green) !important; - } - - &.purple { - color: var(--color-purple) !important; - } - - &.yellow { - color: var(--color-yellow) !important; - } - - &.orange { - color: var(--color-orange) !important; - } - - &.gold { - color: var(--color-gold) !important; - } - &.left { text-align: left !important; } diff --git a/web_src/less/_repository.less b/web_src/less/_repository.less index 039c471309..38103c5f8e 100644 --- a/web_src/less/_repository.less +++ b/web_src/less/_repository.less @@ -829,7 +829,7 @@ .timeline-avatar { position: absolute; - left: -72px; + left: -68px; img { width: 40px !important; @@ -846,7 +846,6 @@ .avatar img { width: 20px; height: 20px; - margin: 0 .25rem; vertical-align: middle; } @@ -981,10 +980,6 @@ margin-top: 4px; } - .author { - font-weight: 600; - } - .comment-form-reply .footer { padding-bottom: 1em; } @@ -1165,9 +1160,12 @@ padding-left: 15px; .detail { - font-size: .9rem; - margin-top: 5px; - margin-left: 8px; + margin-top: 4px; + margin-left: 14px; + + .svg { + margin-right: 2px; + } } .segments { @@ -2673,12 +2671,10 @@ a { color: var(--color-text); - text-decoration: none; } a:hover { color: var(--color-primary); - text-decoration: none; } } diff --git a/web_src/less/shared/issuelist.less b/web_src/less/shared/issuelist.less index 2dffe19909..8d4cfc46d4 100644 --- a/web_src/less/shared/issuelist.less +++ b/web_src/less/shared/issuelist.less @@ -91,6 +91,14 @@ border-radius: 3px; vertical-align: 2px !important; } + + progress::-webkit-progress-value { + background-color: var(--color-secondary-dark-4); + } + + progress::-moz-progress-bar { + background-color: var(--color-secondary-dark-4); + } } .conflicting { diff --git a/web_src/less/themes/theme-arc-green.less b/web_src/less/themes/theme-arc-green.less index 229ffd8937..b793f99509 100644 --- a/web_src/less/themes/theme-arc-green.less +++ b/web_src/less/themes/theme-arc-green.less @@ -56,34 +56,35 @@ --color-secondary-alpha-80: #454a57cc; --color-secondary-alpha-90: #454a57e1; /* colors */ - --color-red: #7d3434; + --color-red: #cc4848; --color-orange: #cc580c; --color-yellow: #cc9903; --color-olive: #91a313; --color-green: #87ab63; --color-teal: #00918a; - --color-blue: #1a6aa6; - --color-violet: #502aa1; - --color-purple: #8229a0; - --color-pink: #c21e7b; - --color-brown: #845232; - --color-grey: #5e626a; - /* light variants */ - --color-red-light: #984646; - --color-orange-light: #e6630d; - --color-yellow-light: #e5ac04; - --color-olive-light: #a3b816; - --color-green-light: #9fbc82; - --color-teal-light: #00a39c; - --color-blue-light: #1e78bb; - --color-violet-light: #5a30b5; - --color-purple-light: #932eb4; - --color-pink-light: #db228a; - --color-brown-light: #955d39; - --color-grey-light: #6a6e78; - /* other colors */ + --color-blue: #3a8ac6; + --color-violet: #906ae1; + --color-purple: #b259d0; + --color-pink: #d22e8b; + --color-brown: #a47252; + --color-grey: #9ea2aa; --color-black: #1e222e; - --color-gold: #a1882b; + /* light variants - produced via Sass scale-color(color, $lightness: -10%) */ + --color-red-light: #c23636; + --color-orange-light: #b84f0b; + --color-yellow-light: #b88a03; + --color-olive-light: #839311; + --color-green-light: #7a9e55; + --color-teal-light: #00837c; + --color-blue-light: #347cb3; + --color-violet-light: #7b4edb; + --color-purple-light: #a742c9; + --color-pink-light: #be297d; + --color-brown-light: #94674a; + --color-grey-light: #8d919b; + --color-black-light: #1b1f29; + /* other colors */ + --color-gold: #b1983b; --color-white: #ffffff; --color-diff-removed-word-bg: #6f3333; --color-diff-added-word-bg: #3c653c; From b2369830bbb9059301011ca41802e76bfc2739c1 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 20 Nov 2022 20:37:20 +0100 Subject: [PATCH 026/182] Do not allow Ghost access to limited visible user/org (#21849) (#21876) Backport of #21849 --- models/organization/org.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/organization/org.go b/models/organization/org.go index 993ca3f10d..217fa623bf 100644 --- a/models/organization/org.go +++ b/models/organization/org.go @@ -458,8 +458,9 @@ func CountOrgs(opts FindOrgOptions) (int64, error) { // HasOrgOrUserVisible tells if the given user can see the given org or user func HasOrgOrUserVisible(ctx context.Context, orgOrUser, user *user_model.User) bool { - // Not SignedUser - if user == nil { + // If user is nil, it's an anonymous user/request. + // The Ghost user is handled like an anonymous user. + if user == nil || user.IsGhost() { return orgOrUser.Visibility == structs.VisibleTypePublic } From f4ec03a4e5b9b2f145227d7f00e90a559c33cdfa Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 22 Nov 2022 02:00:42 +0100 Subject: [PATCH 027/182] Fix setting HTTP headers after write (#21833) (#21877) Backport of #21833 Co-authored-by: techknowlogick --- modules/context/context.go | 66 +++++++++++++++-------- routers/api/packages/rubygems/rubygems.go | 8 ++- routers/common/repo.go | 49 +++++++---------- routers/web/feed/profile.go | 2 - routers/web/web.go | 5 +- 5 files changed, 72 insertions(+), 58 deletions(-) diff --git a/modules/context/context.go b/modules/context/context.go index 4b6a21b217..697eb76904 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -34,6 +34,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/services/auth" @@ -322,9 +323,9 @@ func (ctx *Context) plainTextInternal(skip, status int, bs []byte) { if statusPrefix == 4 || statusPrefix == 5 { log.Log(skip, log.TRACE, "plainTextInternal (status=%d): %s", status, string(bs)) } - ctx.Resp.WriteHeader(status) ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff") + ctx.Resp.WriteHeader(status) if _, err := ctx.Resp.Write(bs); err != nil { log.ErrorWithSkip(skip, "plainTextInternal (status=%d): write bytes failed: %v", status, err) } @@ -345,36 +346,55 @@ func (ctx *Context) RespHeader() http.Header { return ctx.Resp.Header() } +type ServeHeaderOptions struct { + ContentType string // defaults to "application/octet-stream" + ContentTypeCharset string + Disposition string // defaults to "attachment" + Filename string + CacheDuration time.Duration // defaults to 5 minutes +} + // SetServeHeaders sets necessary content serve headers -func (ctx *Context) SetServeHeaders(filename string) { - ctx.Resp.Header().Set("Content-Description", "File Transfer") - ctx.Resp.Header().Set("Content-Type", "application/octet-stream") - ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+filename) - ctx.Resp.Header().Set("Content-Transfer-Encoding", "binary") - ctx.Resp.Header().Set("Expires", "0") - ctx.Resp.Header().Set("Cache-Control", "must-revalidate") - ctx.Resp.Header().Set("Pragma", "public") - ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") +func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { + header := ctx.Resp.Header() + + contentType := typesniffer.ApplicationOctetStream + if opts.ContentType != "" { + if opts.ContentTypeCharset != "" { + contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset) + } else { + contentType = opts.ContentType + } + } + header.Set("Content-Type", contentType) + header.Set("X-Content-Type-Options", "nosniff") + + if opts.Filename != "" { + disposition := opts.Disposition + if disposition == "" { + disposition = "attachment" + } + + backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \" + header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename))) + header.Set("Access-Control-Expose-Headers", "Content-Disposition") + } + + duration := opts.CacheDuration + if duration == 0 { + duration = 5 * time.Minute + } + httpcache.AddCacheControlToHeader(header, duration) } // ServeContent serves content to http request func (ctx *Context) ServeContent(name string, r io.ReadSeeker, modTime time.Time) { - ctx.SetServeHeaders(name) + ctx.SetServeHeaders(&ServeHeaderOptions{ + Filename: name, + }) http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r) } -// ServeFile serves given file to response. -func (ctx *Context) ServeFile(file string, names ...string) { - var name string - if len(names) > 0 { - name = names[0] - } else { - name = path.Base(file) - } - ctx.SetServeHeaders(name) - http.ServeFile(ctx.Resp, ctx.Req, file) -} - // UploadStream returns the request body or the first form file // Only form files need to get closed. func (ctx *Context) UploadStream() (rd io.ReadCloser, needToClose bool, err error) { diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index 319c94b91f..a13810ccdb 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -77,7 +77,9 @@ func enumeratePackages(ctx *context.Context, filename string, pvs []*packages_mo }) } - ctx.SetServeHeaders(filename + ".gz") + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: filename + ".gz", + }) zw := gzip.NewWriter(ctx.Resp) defer zw.Close() @@ -115,7 +117,9 @@ func ServePackageSpecification(ctx *context.Context) { return } - ctx.SetServeHeaders(filename) + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: filename, + }) zw := zlib.NewWriter(ctx.Resp) defer zw.Close() diff --git a/routers/common/repo.go b/routers/common/repo.go index a9e80fad48..f4b813d6b4 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -7,7 +7,6 @@ package common import ( "fmt" "io" - "net/url" "path" "path/filepath" "strings" @@ -53,50 +52,44 @@ func ServeData(ctx *context.Context, filePath string, size int64, reader io.Read buf = buf[:n] } - httpcache.AddCacheControlToHeader(ctx.Resp.Header(), 5*time.Minute) - if size >= 0 { ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", size)) } else { log.Error("ServeData called to serve data: %s with size < 0: %d", filePath, size) } - fileName := path.Base(filePath) - sniffedType := typesniffer.DetectContentType(buf) - isPlain := sniffedType.IsText() || ctx.FormBool("render") - mimeType := "" - charset := "" - - if setting.MimeTypeMap.Enabled { - fileExtension := strings.ToLower(filepath.Ext(fileName)) - mimeType = setting.MimeTypeMap.Map[fileExtension] + opts := &context.ServeHeaderOptions{ + Filename: path.Base(filePath), } - if mimeType == "" { + sniffedType := typesniffer.DetectContentType(buf) + isPlain := sniffedType.IsText() || ctx.FormBool("render") + + if setting.MimeTypeMap.Enabled { + fileExtension := strings.ToLower(filepath.Ext(filePath)) + opts.ContentType = setting.MimeTypeMap.Map[fileExtension] + } + + if opts.ContentType == "" { if sniffedType.IsBrowsableBinaryType() { - mimeType = sniffedType.GetMimeType() + opts.ContentType = sniffedType.GetMimeType() } else if isPlain { - mimeType = "text/plain" + opts.ContentType = "text/plain" } else { - mimeType = typesniffer.ApplicationOctetStream + opts.ContentType = typesniffer.ApplicationOctetStream } } if isPlain { + var charset string charset, err = charsetModule.DetectEncoding(buf) if err != nil { log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err) charset = "utf-8" } + opts.ContentTypeCharset = strings.ToLower(charset) } - if charset != "" { - ctx.Resp.Header().Set("Content-Type", mimeType+"; charset="+strings.ToLower(charset)) - } else { - ctx.Resp.Header().Set("Content-Type", mimeType) - } - ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff") - isSVG := sniffedType.IsSvgImage() // serve types that can present a security risk with CSP @@ -109,16 +102,12 @@ func ServeData(ctx *context.Context, filePath string, size int64, reader io.Read ctx.Resp.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'") } - disposition := "inline" + opts.Disposition = "inline" if isSVG && !setting.UI.SVG.Enabled { - disposition = "attachment" + opts.Disposition = "attachment" } - // encode filename per https://datatracker.ietf.org/doc/html/rfc5987 - encodedFileName := `filename*=UTF-8''` + url.PathEscape(fileName) - - ctx.Resp.Header().Set("Content-Disposition", disposition+"; "+encodedFileName) - ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") + ctx.SetServeHeaders(opts) _, err = ctx.Resp.Write(buf) if err != nil { diff --git a/routers/web/feed/profile.go b/routers/web/feed/profile.go index 0e11f210ce..ffa34572bc 100644 --- a/routers/web/feed/profile.go +++ b/routers/web/feed/profile.go @@ -5,7 +5,6 @@ package feed import ( - "net/http" "time" activities_model "code.gitea.io/gitea/models/activities" @@ -59,7 +58,6 @@ func showUserFeed(ctx *context.Context, formatType string) { // writeFeed write a feeds.Feed as atom or rss to ctx.Resp func writeFeed(ctx *context.Context, feed *feeds.Feed, formatType string) { - ctx.Resp.WriteHeader(http.StatusOK) if formatType == "atom" { ctx.Resp.Header().Set("Content-Type", "application/atom+xml;charset=utf-8") if err := feed.WriteAtom(ctx.Resp); err != nil { diff --git a/routers/web/web.go b/routers/web/web.go index 9b814c3f54..9e0440d09a 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -597,7 +597,10 @@ func RegisterRoutes(m *web.Route) { m.Group("", func() { m.Get("/favicon.ico", func(ctx *context.Context) { - ctx.ServeFile(path.Join(setting.StaticRootPath, "public/img/favicon.png")) + ctx.SetServeHeaders(&context.ServeHeaderOptions{ + Filename: "favicon.png", + }) + http.ServeFile(ctx.Resp, ctx.Req, path.Join(setting.StaticRootPath, "public/img/favicon.png")) }) m.Group("/{username}", func() { m.Get(".png", func(ctx *context.Context) { ctx.Error(http.StatusNotFound) }) From c844c4ff885937ffc2ab0ed5b8a618b5a8df3a46 Mon Sep 17 00:00:00 2001 From: Xinyu Zhou Date: Thu, 24 Nov 2022 10:52:20 +0800 Subject: [PATCH 028/182] Fix vertical align of committer avatar rendered by email address (#21884) (#21918) Backport #21884 Committer avatar rendered by `func AvatarByEmail` are not vertical align as `func Avatar` does. - Replace literals `ui avatar` and `ui avatar vm` with the constant `DefaultAvatarClass` Signed-off-by: Xinyu Zhou --- models/avatars/avatar.go | 8 ++++++-- modules/templates/helper.go | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/models/avatars/avatar.go b/models/avatars/avatar.go index ec3b611312..30bcad033c 100644 --- a/models/avatars/avatar.go +++ b/models/avatars/avatar.go @@ -20,8 +20,12 @@ import ( "code.gitea.io/gitea/modules/setting" ) -// DefaultAvatarPixelSize is the default size in pixels of a rendered avatar -const DefaultAvatarPixelSize = 28 +const ( + // DefaultAvatarClass is the default class of a rendered avatar + DefaultAvatarClass = "ui avatar vm" + // DefaultAvatarPixelSize is the default size in pixels of a rendered avatar + DefaultAvatarPixelSize = 28 +) // EmailHash represents a pre-generated hash map (mainly used by LibravatarURL, it queries email server's DNS records) type EmailHash struct { diff --git a/modules/templates/helper.go b/modules/templates/helper.go index dc90907e01..9aca94971e 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -646,7 +646,7 @@ func SVG(icon string, others ...interface{}) template.HTML { // Avatar renders user avatars. args: user, size (int), class (string) func Avatar(item interface{}, others ...interface{}) template.HTML { - size, class := parseOthers(avatars.DefaultAvatarPixelSize, "ui avatar vm", others...) + size, class := parseOthers(avatars.DefaultAvatarPixelSize, avatars.DefaultAvatarClass, others...) switch t := item.(type) { case *user_model.User: @@ -677,7 +677,7 @@ func AvatarByAction(action *activities_model.Action, others ...interface{}) temp // RepoAvatar renders repo avatars. args: repo, size(int), class (string) func RepoAvatar(repo *repo_model.Repository, others ...interface{}) template.HTML { - size, class := parseOthers(avatars.DefaultAvatarPixelSize, "ui avatar", others...) + size, class := parseOthers(avatars.DefaultAvatarPixelSize, avatars.DefaultAvatarClass, others...) src := repo.RelAvatarLink() if src != "" { @@ -688,7 +688,7 @@ func RepoAvatar(repo *repo_model.Repository, others ...interface{}) template.HTM // AvatarByEmail renders avatars by email address. args: email, name, size (int), class (string) func AvatarByEmail(email, name string, others ...interface{}) template.HTML { - size, class := parseOthers(avatars.DefaultAvatarPixelSize, "ui avatar", others...) + size, class := parseOthers(avatars.DefaultAvatarPixelSize, avatars.DefaultAvatarClass, others...) src := avatars.GenerateEmailAvatarFastLink(email, size*setting.Avatar.RenderedSizeFactor) if src != "" { From 16772ffde3cf33a2ca3e2ede3725cbc5fa4a16ca Mon Sep 17 00:00:00 2001 From: Xinyu Zhou Date: Thu, 24 Nov 2022 14:44:07 +0800 Subject: [PATCH 029/182] Fix flex layout for repo list icons (#21896) (#21920) Backport #21896 #20241 Added a tooltip, which does not satisfy the flex layout, and the icons are not aligned Signed-off-by: Xinyu Zhou --- templates/explore/repo_list.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/explore/repo_list.tmpl b/templates/explore/repo_list.tmpl index 044af9522e..ad03ff800f 100644 --- a/templates/explore/repo_list.tmpl +++ b/templates/explore/repo_list.tmpl @@ -32,9 +32,9 @@ {{end}} {{end}} {{if .IsFork}} - {{svg "octicon-repo-forked"}} + {{svg "octicon-repo-forked"}} {{else if .IsMirror}} - {{svg "octicon-mirror"}} + {{svg "octicon-mirror"}} {{end}}
From 9bccc60cf51f3b4070f5506b042a3d9a1442c73d Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 24 Nov 2022 06:49:22 -0500 Subject: [PATCH 030/182] add changelog for 1.18.0-rc1 (#21829) Co-authored-by: Lunny Xiao --- CHANGELOG.md | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1150d70081..076b2e6e05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,182 @@ This changelog goes through all the changes that have been made in each release without substantial changes to our git log; to see the highlights of what has been added to each release, please refer to the [blog](https://blog.gitea.io). +## [1.18.0-rc1](https://github.com/go-gitea/gitea/releases/tag/v1.18.0-rc1) - 2022-11-15 + +* BREAKING + * Remove U2F support (#20141) +* FEATURES + * Add color previews in markdown (#21474) + * Allow package version sorting (#21453) + * Add support for Chocolatey/NuGet v2 API (#21393) + * Add API endpoint to get changed files of a PR (#21177) + * Add filetree on left of diff view (#21012) + * Support Issue forms and PR forms (#20987) + * Add support for Vagrant packages (#20930) + * Add support for `npm unpublish` (#20688) + * Add badge capabilities to users (#20607) + * Add issue filter for Author (#20578) + * Add KaTeX rendering to Markdown. (#20571) + * Add support for Pub packages (#20560) + * Support localized README (#20508) + * Add support mCaptcha as captcha provider (#20458) + * Add team member invite by email (#20307) + * Added email notification option to receive all own messages (#20179) + * Switch Unicode Escaping to a VSCode-like system (#19990) + * Add user/organization code search (#19977) + * Only show relevant repositories on explore page (#19361) + * User keypairs and HTTP signatures for ActivityPub federation using go-ap (#19133) + * Add sitemap support (#18407) + * Allow creation of OAuth2 applications for orgs (#18084) + * Add system setting table with cache and also add cache supports for user setting (#18058) + * Add pages to view watched repos and subscribed issues/PRs (#17156) + * Support Proxy protocol (#12527) + * Implement sync push mirror on commit (#19411) +* API + * Make external issue tracker regexp configurable via API (#21338) + * Add name field for org api (#21270) + * Show teams with no members if user is admin (#21204) + * Add latest commit's SHA to content response (#20398) + * Add allow_rebase_update, default_delete_branch_after_merge to repository api response (#20079) + * Add new endpoints for push mirrors management (#19841) +* ENHANCEMENTS + * Use CSS color-scheme instead of invert (#21616) (#21623) + * Respect user's locale when rendering the date range in the repo activity page (#21410) + * Change `commits-table` column width (#21564) + * Refactor git command arguments and make all arguments to be safe to be used (#21535) + * CSS color enhancements (#21534) + * Add link to user profile in markdown mention only if user exists (#21533, #21554) + * Add option to skip index dirs (#21501) + * Diff file tree tweaks (#21446) + * Localize all timestamps (#21440) + * Add `code` highlighting in issue titles (#21432) + * Use Name instead of DisplayName in LFS Lock (#21415) + * Consolidate more CSS colors into variables (#21402) + * Redirect to new repository owner (#21398) + * Use ISO date format instead of hard-coded English date format for date range in repo activity page (#21396) + * Use weighted algorithm for string matching when finding files in repo (#21370) + * Show private data in feeds (#21369) + * Refactor parseTreeEntries, speed up tree list (#21368) + * Add GET and DELETE endpoints for Docker blob uploads (#21367) + * Add nicer error handling on template compile errors (#21350) + * Add `stat` to `ToCommit` function for speed (#21337) + * Support instance-wide OAuth2 applications (#21335) + * Record OAuth client type at registration (#21316) + * Add new CSS variables --color-accent and --color-small-accent (#21305) + * Improve error descriptions for unauthorized_client (#21292) + * Case-insensitive "find files in repo" (#21269) + * Consolidate more CSS rules, fix inline code on arc-green (#21260) + * Log real ip of requests from ssh (#21216) + * Save files in local storage as group readable (#21198) + * Enable fluid page layout on medium size viewports (#21178) + * File header tweaks (#21175) + * Added missing headers on user packages page (#21172) + * Display image digest for container packages (#21170) + * Skip dirty check for team forms (#21154) + * Keep path when creating a new branch (#21153) + * Remove fomantic image module (#21145) + * Make labels clickable in the comments section. (#21137) + * Sort branches and tags by date descending (#21136) + * Better repo API unit checks (#21130) + * Improve commit status icons (#21124) + * Limit length of repo description and repo url input fields (#21119) + * Show .editorconfig errors in frontend (#21088) + * Allow poster to choose reviewers (#21084) + * Remove black labels and CSS cleanup (#21003) + * Make e-mail sanity check more precise (#20991) + * Use native inputs in whitespace dropdown (#20980) + * Enhance package date display (#20928) + * Display total blob size of a package version (#20927) + * Show language name on hover (#20923) + * Show instructions for all generic package files (#20917) + * Refactor AssertExistsAndLoadBean to use generics (#20797) + * Move the official website link at the footer of gitea (#20777) + * Add support for full name in reverse proxy auth (#20776) + * Remove useless JS operation for relative time tooltips (#20756) + * Replace some icons with SVG (#20741) + * Change commit status icons to SVG (#20736) + * Improve single repo action for issue and pull requests (#20730) + * Allow multiple files in generic packages (#20661) + * Add option to create new issue from /issues page (#20650) + * Background color of private list-items updated (#20630) + * Added search input field to issue filter (#20623) + * Increase default item listing size `ISSUE_PAGING_NUM` to 20 (#20547) + * Modify milestone search keywords to be case insensitive again (#20513) + * Show hint to link package to repo when viewing empty repo package list (#20504) + * Add Tar ZSTD support (#20493) + * Make code review checkboxes clickable (#20481) + * Add "X-Gitea-Object-Type" header for GET `/raw/` & `/media/` API (#20438) + * Display project in issue list (#20434) + * Prepend commit message to template content when opening a new PR (#20429) + * Replace fomantic popup module with tippy.js (#20428) + * Allow to specify colors for text in markup (#20363) + * Allow access to the Public Organization Member lists with minimal permissions (#20330) + * Use default values when provided values are empty (#20318) + * Vertical align navbar avatar at middle (#20302) + * Delete cancel button in repo creation page (#21381) + * Include login_name in adminCreateUser response (#20283) + * fix: icon margin in user/settings/repos (#20281) + * Remove blue text on migrate page (#20273) + * Modify milestone search keywords to be case insensitive (#20266) + * Move some files into models' sub packages (#20262) + * Add tooltip to repo icons in explore page (#20241) + * Remove deprecated licenses (#20222) + * Webhook for Wiki changes (#20219) + * Share HTML template renderers and create a watcher framework (#20218) + * Allow enable LDAP source and disable user sync via CLI (#20206) + * Adds a checkbox to select all issues/PRs (#20177) + * Refactor `i18n` to `locale` (#20153) + * Disable status checks in template if none found (#20088) + * Allow manager logging to set SQL (#20064) + * Add order by for assignee no sort issue (#20053) + * Take a stab at porting existing components to Vue3 (#20044) + * Add doctor command to write commit-graphs (#20007) + * Add support for authentication based on reverse proxy email (#19949) + * Enable spellcheck for EasyMDE, use contenteditable mode (#19776) + * Allow specifying SECRET_KEY_URI, similar to INTERNAL_TOKEN_URI (#19663) + * Rework mailer settings (#18982) + * Add option to purge users (#18064) + * Add author search input (#21246) + * Make rss/atom identifier globally unique (#21550) +* BUGFIXES + * Prevent panic in doctor command when running default checks (#21791) (#21807) + * Load GitRepo in API before deleting issue (#21720) (#21796) + * Ignore line anchor links with leading zeroes (#21728) (#21776) + * Set last login when activating account (#21731) (#21755) + * Fix UI language switching bug (#21597) (#21749) + * Quick fixes monaco-editor error: "vs.editor.nullLanguage" (#21734) (#21738) + * Allow local package identifiers for PyPI packages (#21690) (#21727) + * Deal with markdown template without metadata (#21639) (#21654) + * Fix opaque background on mermaid diagrams (#21642) (#21652) + * Fix repository adoption on Windows (#21646) (#21650) + * Sync git hooks when config file path changed (#21619) (#21626) + * Fix 500 on PR files API (#21602) (#21607) + * Fix `Timestamp.IsZero` (#21593) (#21603) + * Fix viewing user subscriptions (#21482) + * Fix mermaid-related bugs (#21431) + * Fix branch dropdown shifting on page load (#21428) + * Fix default theme-auto selector when nologin (#21346) + * Fix and improve incorrect error messages (#21342) + * Fix formatted link for PR review notifications to matrix (#21319) + * Center-aligning content of WebAuthN page (#21127) + * Remove follow from commits by file (#20765) + * Fix commit status popup (#20737) + * Fix init mail render logic (#20704) + * Use correct page size for link header pagination (#20546) + * Preserve unix socket file (#20499) + * Use tippy.js for context popup (#20393) + * Add missing parameter for error in log message (#20144) + * Do not allow organisation owners add themselves as collaborator (#20043) + * Rework file highlight rendering and fix yaml copy-paste (#19967) + * Improve code diff highlight, fix incorrect rendered diff result (#19958) +* TESTING + * Improve OAuth integration tests (#21390) + * Add playwright tests (#20123) +* BUILD + * Switch to building with go1.19 (#20695) + * Update JS dependencies, adjust eslint (#20659) + * Add more linters to improve code readability (#19989) + ## [1.17.3](https://github.com/go-gitea/gitea/releases/tag/v1.17.3) - 2022-10-15 * SECURITY From 9ba4ef93ff114588f2392ee130a19bdeae3c61da Mon Sep 17 00:00:00 2001 From: Xinyu Zhou Date: Fri, 25 Nov 2022 00:02:33 +0800 Subject: [PATCH 031/182] Fix button in branch list, avoid unexpected page jump before restore branch actually done (#21562) (#21928) Backport #21562 Signed-off-by: Xinyu Zhou Co-authored-by: Lunny Xiao Co-authored-by: Lauris BH --- templates/repo/branch/list.tmpl | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/templates/repo/branch/list.tmpl b/templates/repo/branch/list.tmpl index 09799fbece..3702410df5 100644 --- a/templates/repo/branch/list.tmpl +++ b/templates/repo/branch/list.tmpl @@ -22,18 +22,18 @@ {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} -
+
+ {{end}} {{if not $.DisableDownloadSourceArchives}} -
+
+ {{end}} @@ -108,26 +108,30 @@ {{if and $.IsWriter (not $.Repository.IsArchived) (not .IsDeleted)}} -
+
+ {{end}} {{if and (not .IsDeleted) (not $.DisableDownloadSourceArchives)}} -
+
+ {{end}} {{if and $.IsWriter (not $.IsMirror) (not $.Repository.IsArchived) (not .IsProtected)}} {{if .IsDeleted}} - {{svg "octicon-reply"}} + {{else}} - + {{end}} {{end}} From ff4e292b3fe49ee3086f45de1717301981e5f5f2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 25 Nov 2022 12:46:28 +0100 Subject: [PATCH 032/182] Add support for HEAD requests in Maven registry (#21834) (#21929) Backport of #21834 Co-authored-by: Lunny Xiao --- modules/context/context.go | 18 +++++-- routers/api/packages/api.go | 1 + routers/api/packages/composer/composer.go | 5 +- routers/api/packages/conan/conan.go | 5 +- routers/api/packages/generic/generic.go | 5 +- routers/api/packages/helm/helm.go | 5 +- routers/api/packages/maven/api.go | 7 +-- routers/api/packages/maven/maven.go | 57 ++++++++++++++++++-- routers/api/packages/npm/npm.go | 10 +++- routers/api/packages/nuget/nuget.go | 10 +++- routers/api/packages/pub/pub.go | 5 +- routers/api/packages/pypi/pypi.go | 5 +- routers/api/packages/rubygems/rubygems.go | 5 +- routers/api/packages/vagrant/vagrant.go | 5 +- routers/api/v1/repo/file.go | 6 ++- routers/common/repo.go | 13 +++-- routers/web/repo/repo.go | 5 +- routers/web/user/package.go | 5 +- tests/integration/api_packages_maven_test.go | 30 ++++++++++- 19 files changed, 162 insertions(+), 40 deletions(-) diff --git a/modules/context/context.go b/modules/context/context.go index 697eb76904..47368bb280 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -349,9 +349,11 @@ func (ctx *Context) RespHeader() http.Header { type ServeHeaderOptions struct { ContentType string // defaults to "application/octet-stream" ContentTypeCharset string + ContentLength *int64 Disposition string // defaults to "attachment" Filename string CacheDuration time.Duration // defaults to 5 minutes + LastModified time.Time } // SetServeHeaders sets necessary content serve headers @@ -369,6 +371,10 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { header.Set("Content-Type", contentType) header.Set("X-Content-Type-Options", "nosniff") + if opts.ContentLength != nil { + header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10)) + } + if opts.Filename != "" { disposition := opts.Disposition if disposition == "" { @@ -385,14 +391,16 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { duration = 5 * time.Minute } httpcache.AddCacheControlToHeader(header, duration) + + if !opts.LastModified.IsZero() { + header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat)) + } } // ServeContent serves content to http request -func (ctx *Context) ServeContent(name string, r io.ReadSeeker, modTime time.Time) { - ctx.SetServeHeaders(&ServeHeaderOptions{ - Filename: name, - }) - http.ServeContent(ctx.Resp, ctx.Req, name, modTime, r) +func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) { + ctx.SetServeHeaders(opts) + http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r) } // UploadStream returns the request body or the first form file diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go index 6f53bc4ae0..2b3d7de3d1 100644 --- a/routers/api/packages/api.go +++ b/routers/api/packages/api.go @@ -179,6 +179,7 @@ func Routes(ctx gocontext.Context) *web.Route { r.Group("/maven", func() { r.Put("/*", reqPackageAccess(perm.AccessModeWrite), maven.UploadPackageFile) r.Get("/*", maven.DownloadPackageFile) + r.Head("/*", maven.ProvidePackageFileHeader) }, reqPackageAccess(perm.AccessModeRead)) r.Group("/nuget", func() { r.Group("", func() { // Needs to be unauthenticated for the NuGet client. diff --git a/routers/api/packages/composer/composer.go b/routers/api/packages/composer/composer.go index 86ef7cbd9a..476a2c236a 100644 --- a/routers/api/packages/composer/composer.go +++ b/routers/api/packages/composer/composer.go @@ -184,7 +184,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/conan/conan.go b/routers/api/packages/conan/conan.go index dd078d6ad3..ac99a48e98 100644 --- a/routers/api/packages/conan/conan.go +++ b/routers/api/packages/conan/conan.go @@ -473,7 +473,10 @@ func downloadFile(ctx *context.Context, fileFilter container.Set[string], fileKe } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DeleteRecipeV1 deletes the requested recipe(s) diff --git a/routers/api/packages/generic/generic.go b/routers/api/packages/generic/generic.go index 81891bec26..f2bc1dc597 100644 --- a/routers/api/packages/generic/generic.go +++ b/routers/api/packages/generic/generic.go @@ -53,7 +53,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage uploads the specific generic package. diff --git a/routers/api/packages/helm/helm.go b/routers/api/packages/helm/helm.go index 9c85e0874f..43dafe6296 100644 --- a/routers/api/packages/helm/helm.go +++ b/routers/api/packages/helm/helm.go @@ -138,7 +138,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/maven/api.go b/routers/api/packages/maven/api.go index b60a317814..4ca541dd6f 100644 --- a/routers/api/packages/maven/api.go +++ b/routers/api/packages/maven/api.go @@ -6,7 +6,6 @@ package maven import ( "encoding/xml" - "sort" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -23,12 +22,8 @@ type MetadataResponse struct { Version []string `xml:"versioning>versions>version"` } +// pds is expected to be sorted ascending by CreatedUnix func createMetadataResponse(pds []*packages_model.PackageDescriptor) *MetadataResponse { - sort.Slice(pds, func(i, j int) bool { - // Maven and Gradle order packages by their creation timestamp and not by their version string - return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix - }) - var release *packages_model.PackageDescriptor versions := make([]string, 0, len(pds)) diff --git a/routers/api/packages/maven/maven.go b/routers/api/packages/maven/maven.go index bf00c199f5..de88328806 100644 --- a/routers/api/packages/maven/maven.go +++ b/routers/api/packages/maven/maven.go @@ -16,6 +16,8 @@ import ( "net/http" "path/filepath" "regexp" + "sort" + "strconv" "strings" packages_model "code.gitea.io/gitea/models/packages" @@ -34,6 +36,10 @@ const ( extensionSHA1 = ".sha1" extensionSHA256 = ".sha256" extensionSHA512 = ".sha512" + extensionPom = ".pom" + extensionJar = ".jar" + contentTypeJar = "application/java-archive" + contentTypeXML = "text/xml" ) var ( @@ -49,6 +55,15 @@ func apiError(ctx *context.Context, status int, obj interface{}) { // DownloadPackageFile serves the content of a package func DownloadPackageFile(ctx *context.Context) { + handlePackageFile(ctx, true) +} + +// ProvidePackageFileHeader provides only the headers describing a package +func ProvidePackageFileHeader(ctx *context.Context) { + handlePackageFile(ctx, false) +} + +func handlePackageFile(ctx *context.Context, serveContent bool) { params, err := extractPathParameters(ctx) if err != nil { apiError(ctx, http.StatusBadRequest, err) @@ -58,7 +73,7 @@ func DownloadPackageFile(ctx *context.Context) { if params.IsMeta && params.Version == "" { serveMavenMetadata(ctx, params) } else { - servePackageFile(ctx, params) + servePackageFile(ctx, params, serveContent) } } @@ -82,6 +97,11 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { return } + sort.Slice(pds, func(i, j int) bool { + // Maven and Gradle order packages by their creation timestamp and not by their version string + return pds[i].Version.CreatedUnix < pds[j].Version.CreatedUnix + }) + xmlMetadata, err := xml.Marshal(createMetadataResponse(pds)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -89,6 +109,9 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { } xmlMetadataWithHeader := append([]byte(xml.Header), xmlMetadata...) + latest := pds[len(pds)-1] + ctx.Resp.Header().Set("Last-Modified", latest.Version.CreatedUnix.Format(http.TimeFormat)) + ext := strings.ToLower(filepath.Ext(params.Filename)) if isChecksumExtension(ext) { var hash []byte @@ -110,10 +133,15 @@ func serveMavenMetadata(ctx *context.Context, params parameters) { return } - ctx.PlainTextBytes(http.StatusOK, xmlMetadataWithHeader) + ctx.Resp.Header().Set("Content-Length", strconv.Itoa(len(xmlMetadataWithHeader))) + ctx.Resp.Header().Set("Content-Type", contentTypeXML) + + if _, err := ctx.Resp.Write(xmlMetadataWithHeader); err != nil { + log.Error("write bytes failed: %v", err) + } } -func servePackageFile(ctx *context.Context, params parameters) { +func servePackageFile(ctx *context.Context, params parameters, serveContent bool) { packageName := params.GroupID + "-" + params.ArtifactID pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeMaven, packageName, params.Version) @@ -165,6 +193,23 @@ func servePackageFile(ctx *context.Context, params parameters) { return } + opts := &context.ServeHeaderOptions{ + ContentLength: &pb.Size, + LastModified: pf.CreatedUnix.AsLocalTime(), + } + switch ext { + case extensionJar: + opts.ContentType = contentTypeJar + case extensionPom: + opts.ContentType = contentTypeXML + } + + if !serveContent { + ctx.SetServeHeaders(opts) + ctx.Status(http.StatusOK) + return + } + s, err := packages_module.NewContentStore().Get(packages_module.BlobHash256Key(pb.HashSHA256)) if err != nil { apiError(ctx, http.StatusInternalServerError, err) @@ -177,7 +222,9 @@ func servePackageFile(ctx *context.Context, params parameters) { } } - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + opts.Filename = pf.Name + + ctx.ServeContent(s, opts) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. @@ -272,7 +319,7 @@ func UploadPackageFile(ctx *context.Context) { } // If it's the package pom file extract the metadata - if ext == ".pom" { + if ext == extensionPom { pfci.IsLead = true var err error diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index 82dae0cf43..a9676b9191 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -103,7 +103,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DownloadPackageFileByName finds the version and serves the contents of a package @@ -146,7 +149,10 @@ func DownloadPackageFileByName(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index e84aef3160..5267bca8f5 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -342,7 +342,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackage creates a new package with the metadata contained in the uploaded nupgk file @@ -552,7 +555,10 @@ func DownloadSymbolFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // DeletePackage hard deletes the package diff --git a/routers/api/packages/pub/pub.go b/routers/api/packages/pub/pub.go index 9af0ceeb0e..672669c985 100644 --- a/routers/api/packages/pub/pub.go +++ b/routers/api/packages/pub/pub.go @@ -271,5 +271,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go index 4c8041c30c..2063fc37ce 100644 --- a/routers/api/packages/pypi/pypi.go +++ b/routers/api/packages/pypi/pypi.go @@ -95,7 +95,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. diff --git a/routers/api/packages/rubygems/rubygems.go b/routers/api/packages/rubygems/rubygems.go index a13810ccdb..d83120ca2f 100644 --- a/routers/api/packages/rubygems/rubygems.go +++ b/routers/api/packages/rubygems/rubygems.go @@ -192,7 +192,10 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } // UploadPackageFile adds a file to the package. If the package does not exist, it gets created. diff --git a/routers/api/packages/vagrant/vagrant.go b/routers/api/packages/vagrant/vagrant.go index 7750e5dc4b..35643e3150 100644 --- a/routers/api/packages/vagrant/vagrant.go +++ b/routers/api/packages/vagrant/vagrant.go @@ -235,5 +235,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go index 6dead81e6d..aba5b1f9e7 100644 --- a/routers/api/v1/repo/file.go +++ b/routers/api/v1/repo/file.go @@ -341,7 +341,11 @@ func download(ctx *context.APIContext, archiveName string, archiver *repo_model. return } defer fr.Close() - ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) + + ctx.ServeContent(fr, &context.ServeHeaderOptions{ + Filename: downloadName, + LastModified: archiver.CreatedUnix.AsLocalTime(), + }) } // GetEditorconfig get editor config of a repository diff --git a/routers/common/repo.go b/routers/common/repo.go index f4b813d6b4..340eb1809f 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -5,7 +5,6 @@ package common import ( - "fmt" "io" "path" "path/filepath" @@ -52,16 +51,16 @@ func ServeData(ctx *context.Context, filePath string, size int64, reader io.Read buf = buf[:n] } - if size >= 0 { - ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", size)) - } else { - log.Error("ServeData called to serve data: %s with size < 0: %d", filePath, size) - } - opts := &context.ServeHeaderOptions{ Filename: path.Base(filePath), } + if size >= 0 { + opts.ContentLength = &size + } else { + log.Error("ServeData called to serve data: %s with size < 0: %d", filePath, size) + } + sniffedType := typesniffer.DetectContentType(buf) isPlain := sniffedType.IsText() || ctx.FormBool("render") diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go index 3e746d3f05..0bad3f5de1 100644 --- a/routers/web/repo/repo.go +++ b/routers/web/repo/repo.go @@ -426,7 +426,10 @@ func download(ctx *context.Context, archiveName string, archiver *repo_model.Rep } defer fr.Close() - ctx.ServeContent(downloadName, fr, archiver.CreatedUnix.AsLocalTime()) + ctx.ServeContent(fr, &context.ServeHeaderOptions{ + Filename: downloadName, + LastModified: archiver.CreatedUnix.AsLocalTime(), + }) } // InitiateDownload will enqueue an archival request, as needed. It may submit diff --git a/routers/web/user/package.go b/routers/web/user/package.go index 7179e2df97..7be37b6a50 100644 --- a/routers/web/user/package.go +++ b/routers/web/user/package.go @@ -402,5 +402,8 @@ func DownloadPackageFile(ctx *context.Context) { } defer s.Close() - ctx.ServeContent(pf.Name, s, pf.CreatedUnix.AsLocalTime()) + ctx.ServeContent(s, &context.ServeHeaderOptions{ + Filename: pf.Name, + LastModified: pf.CreatedUnix.AsLocalTime(), + }) } diff --git a/tests/integration/api_packages_maven_test.go b/tests/integration/api_packages_maven_test.go index 87d95557ce..e71e1ff03b 100644 --- a/tests/integration/api_packages_maven_test.go +++ b/tests/integration/api_packages_maven_test.go @@ -7,6 +7,7 @@ package integration import ( "fmt" "net/http" + "strconv" "strings" "testing" @@ -39,6 +40,12 @@ func TestPackageMaven(t *testing.T) { MakeRequest(t, req, expectedStatus) } + checkHeaders := func(t *testing.T, h http.Header, contentType string, contentLength int64) { + assert.Equal(t, contentType, h.Get("Content-Type")) + assert.Equal(t, strconv.FormatInt(contentLength, 10), h.Get("Content-Length")) + assert.NotEmpty(t, h.Get("Last-Modified")) + } + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -77,10 +84,18 @@ func TestPackageMaven(t *testing.T) { t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) + checkHeaders(t, resp.Header(), "application/java-archive", 4) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + checkHeaders(t, resp.Header(), "application/java-archive", 4) + assert.Equal(t, []byte("test"), resp.Body.Bytes()) pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) @@ -150,10 +165,18 @@ func TestPackageMaven(t *testing.T) { t.Run("DownloadPOM", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) + req := NewRequest(t, "HEAD", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) req = AddBasicAuthHeader(req, user.Name) resp := MakeRequest(t, req, http.StatusOK) + checkHeaders(t, resp.Header(), "text/xml", int64(len(pomContent))) + + req = NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s.pom", root, packageVersion, filename)) + req = AddBasicAuthHeader(req, user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + checkHeaders(t, resp.Header(), "text/xml", int64(len(pomContent))) + assert.Equal(t, []byte(pomContent), resp.Body.Bytes()) pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeMaven) @@ -191,6 +214,9 @@ func TestPackageMaven(t *testing.T) { resp := MakeRequest(t, req, http.StatusOK) expectedMetadata := `` + "\ncom.giteatest-project1.0.11.0.11.0.1" + + checkHeaders(t, resp.Header(), "text/xml", int64(len(expectedMetadata))) + assert.Equal(t, expectedMetadata, resp.Body.String()) for key, checksum := range map[string]string{ From b56d269cf8a36582863f2629f80685087afb5a41 Mon Sep 17 00:00:00 2001 From: zeripath Date: Fri, 25 Nov 2022 23:28:03 +0000 Subject: [PATCH 033/182] Prevent NPE if trying to restore an already restored deleted branch (#21940) (#21944) Backport #21940 If a deleted-branch has already been restored, a request to restore it again will cause a NPE. This PR adds detection for this case, but also disables buttons when they're clicked in order to help prevent accidental repeat requests. Fix #21930 Signed-off-by: Andrew Thornton --- routers/web/repo/branch.go | 4 ++++ web_src/js/features/common-global.js | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index d1f1255db4..6085fd8693 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -126,6 +126,10 @@ func RestoreBranchPost(ctx *context.Context) { log.Error("GetDeletedBranchByID: %v", err) ctx.Flash.Error(ctx.Tr("repo.branch.restore_failed", branchName)) return + } else if deletedBranch == nil { + log.Debug("RestoreBranch: Can't restore branch[%d] '%s', as it does not exist", branchID, branchName) + ctx.Flash.Error(ctx.Tr("repo.branch.restore_failed", branchName)) + return } if err := git.Push(ctx, ctx.Repo.Repository.RepoPath(), git.PushOptions{ diff --git a/web_src/js/features/common-global.js b/web_src/js/features/common-global.js index b00b4aea9c..7efefd7084 100644 --- a/web_src/js/features/common-global.js +++ b/web_src/js/features/common-global.js @@ -260,6 +260,7 @@ export function initGlobalLinkActions() { e.preventDefault(); const $this = $(this); const redirect = $this.data('redirect'); + $this.prop('disabled', true); $.post($this.data('url'), { _csrf: csrfToken }).done((data) => { @@ -270,6 +271,8 @@ export function initGlobalLinkActions() { } else { window.location.reload(); } + }).always(() => { + $this.prop('disabled', false); }); } @@ -283,11 +286,14 @@ export function initGlobalLinkActions() { // FIXME: this is only used once, and should be replace with `link-action` instead $('.undo-button').on('click', function () { const $this = $(this); + $this.prop('disabled', true); $.post($this.data('url'), { _csrf: csrfToken, id: $this.data('id') }).done((data) => { window.location.href = data.redirect; + }).always(() => { + $this.prop('disabled', false); }); }); } From cddceb9dcacc769ffe21470fee928641ba92ec79 Mon Sep 17 00:00:00 2001 From: silverwind Date: Sat, 26 Nov 2022 17:21:13 +0100 Subject: [PATCH 034/182] Fix markdown anchor re-clicking (#21931) (#21946) Backport #21931. The hashchange event did not fire on re-click of a active anchor. Instead, use the click event which always fires. Fixes: https://github.com/go-gitea/gitea/issues/21680 Co-authored-by: Lunny Xiao Co-authored-by: zeripath --- web_src/js/markup/anchors.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/web_src/js/markup/anchors.js b/web_src/js/markup/anchors.js index cc2ed5db78..53dfa2980c 100644 --- a/web_src/js/markup/anchors.js +++ b/web_src/js/markup/anchors.js @@ -2,10 +2,11 @@ import {svg} from '../svg.js'; const headingSelector = '.markup h1, .markup h2, .markup h3, .markup h4, .markup h5, .markup h6'; -function scrollToAnchor() { - if (document.querySelector(':target')) return; - if (!window.location.hash || window.location.hash.length <= 1) return; - const id = decodeURIComponent(window.location.hash.substring(1)); +function scrollToAnchor(hash, initial) { + // abort if the browser has already scrolled to another anchor during page load + if (initial && document.querySelector(':target')) return; + if (hash?.length <= 1) return; + const id = decodeURIComponent(hash.substring(1)); const el = document.getElementById(`user-content-${id}`); if (el) { el.scrollIntoView(); @@ -24,9 +25,11 @@ export function initMarkupAnchors() { a.classList.add('anchor'); a.setAttribute('href', `#${encodeURIComponent(originalId)}`); a.innerHTML = svg('octicon-link'); + a.addEventListener('click', (e) => { + scrollToAnchor(e.currentTarget.getAttribute('href'), false); + }); heading.prepend(a); } - scrollToAnchor(); - window.addEventListener('hashchange', scrollToAnchor); + scrollToAnchor(window.location.hash, true); } From 888384a63181af9183ecfe134961588f1a842b37 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 27 Nov 2022 19:45:59 +0000 Subject: [PATCH 035/182] Correct the fallbacks for mailer configuration (#21945) (#21953) Backport #21945 Unfortunately the fallback configuration code for [mailer] that were added in #18982 are incorrect. When you read a value from an ini section that key is added. This leads to a failure of the fallback mechanism. Further there is also a spelling mistake in the startTLS configuration. This PR restructures the mailer code to first map the deprecated settings on to the new ones - and then use ini.MapTo to map those on to the struct with additional validation as necessary. Ref #21744 Signed-off-by: Andrew Thornton --- custom/conf/app.example.ini | 2 +- .../doc/advanced/config-cheat-sheet.en-us.md | 2 +- modules/setting/mailer.go | 180 +++++++++--------- modules/setting/setting.go | 2 +- services/mailer/mailer.go | 4 +- 5 files changed, 96 insertions(+), 94 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index ec0b7c5235..3f5ce8046f 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1550,7 +1550,7 @@ ROUTER = console ;; Prefix displayed before subject in mail ;SUBJECT_PREFIX = ;; -;; Mail server protocol. One of "smtp", "smtps", "smtp+startls", "smtp+unix", "sendmail", "dummy". +;; Mail server protocol. One of "smtp", "smtps", "smtp+starttls", "smtp+unix", "sendmail", "dummy". ;; - sendmail: use the operating system's `sendmail` command instead of SMTP. This is common on Linux systems. ;; - dummy: send email messages to the log as a testing phase. ;; If your provider does not explicitly say which protocol it uses but does provide a port, diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 3fd853fb14..955b9ac228 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -672,7 +672,7 @@ and [Gitea 1.17 configuration document](https://github.com/go-gitea/gitea/blob/release/v1.17/docs/content/doc/advanced/config-cheat-sheet.en-us.md) - `ENABLED`: **false**: Enable to use a mail service. -- `PROTOCOL`: **\**: Mail server protocol. One of "smtp", "smtps", "smtp+startls", "smtp+unix", "sendmail", "dummy". _Before 1.18, this was inferred from a combination of `MAILER_TYPE` and `IS_TLS_ENABLED`._ +- `PROTOCOL`: **\**: Mail server protocol. One of "smtp", "smtps", "smtp+starttls", "smtp+unix", "sendmail", "dummy". _Before 1.18, this was inferred from a combination of `MAILER_TYPE` and `IS_TLS_ENABLED`._ - SMTP family, if your provider does not explicitly say which protocol it uses but does provide a port, you can set SMTP_PORT instead and this will be inferred. - **sendmail** Use the operating system's `sendmail` command instead of SMTP. This is common on Linux systems. - **dummy** Send email messages to the log as a testing phase. diff --git a/modules/setting/mailer.go b/modules/setting/mailer.go index d6f1dae0f7..5a9b7054b9 100644 --- a/modules/setting/mailer.go +++ b/modules/setting/mailer.go @@ -18,32 +18,33 @@ import ( // Mailer represents mail service. type Mailer struct { // Mailer - Name string - From string - EnvelopeFrom string - OverrideEnvelopeFrom bool `ini:"-"` - FromName string - FromEmail string - SendAsPlainText bool - SubjectPrefix string + Name string `ini:"NAME"` + From string `ini:"FROM"` + EnvelopeFrom string `ini:"ENVELOPE_FROM"` + OverrideEnvelopeFrom bool `ini:"-"` + FromName string `ini:"-"` + FromEmail string `ini:"-"` + SendAsPlainText bool `ini:"SEND_AS_PLAIN_TEXT"` + SubjectPrefix string `ini:"SUBJECT_PREFIX"` // SMTP sender - Protocol string - SMTPAddr string - SMTPPort string - User, Passwd string - EnableHelo bool - HeloHostname string - ForceTrustServerCert bool - UseClientCert bool - ClientCertFile string - ClientKeyFile string + Protocol string `ini:"PROTOCOL"` + SMTPAddr string `ini:"SMTP_ADDR"` + SMTPPort string `ini:"SMTP_PORT"` + User string `ini:"USER"` + Passwd string `ini:"PASSWD"` + EnableHelo bool `ini:"ENABLE_HELO"` + HeloHostname string `ini:"HELO_HOSTNAME"` + ForceTrustServerCert bool `ini:"FORCE_TRUST_SERVER_CERT"` + UseClientCert bool `ini:"USE_CLIENT_CERT"` + ClientCertFile string `ini:"CLIENT_CERT_FILE"` + ClientKeyFile string `ini:"CLIENT_KEY_FILE"` // Sendmail sender - SendmailPath string - SendmailArgs []string - SendmailTimeout time.Duration - SendmailConvertCRLF bool + SendmailPath string `ini:"SENDMAIL_PATH"` + SendmailArgs []string `ini:"-"` + SendmailTimeout time.Duration `ini:"SENDMAIL_TIMEOUT"` + SendmailConvertCRLF bool `ini:"SENDMAIL_CONVERT_CRLF"` } // MailService the global mailer @@ -56,35 +57,12 @@ func newMailService() { return } - MailService = &Mailer{ - Name: sec.Key("NAME").MustString(AppName), - SendAsPlainText: sec.Key("SEND_AS_PLAIN_TEXT").MustBool(false), - - Protocol: sec.Key("PROTOCOL").In("", []string{"smtp", "smtps", "smtp+startls", "smtp+unix", "sendmail", "dummy"}), - SMTPAddr: sec.Key("SMTP_ADDR").String(), - SMTPPort: sec.Key("SMTP_PORT").String(), - User: sec.Key("USER").String(), - Passwd: sec.Key("PASSWD").String(), - EnableHelo: sec.Key("ENABLE_HELO").MustBool(true), - HeloHostname: sec.Key("HELO_HOSTNAME").String(), - ForceTrustServerCert: sec.Key("FORCE_TRUST_SERVER_CERT").MustBool(false), - UseClientCert: sec.Key("USE_CLIENT_CERT").MustBool(false), - ClientCertFile: sec.Key("CLIENT_CERT_FILE").String(), - ClientKeyFile: sec.Key("CLIENT_KEY_FILE").String(), - SubjectPrefix: sec.Key("SUBJECT_PREFIX").MustString(""), - - SendmailPath: sec.Key("SENDMAIL_PATH").MustString("sendmail"), - SendmailTimeout: sec.Key("SENDMAIL_TIMEOUT").MustDuration(5 * time.Minute), - SendmailConvertCRLF: sec.Key("SENDMAIL_CONVERT_CRLF").MustBool(true), - } - MailService.From = sec.Key("FROM").MustString(MailService.User) - MailService.EnvelopeFrom = sec.Key("ENVELOPE_FROM").MustString("") - + // Handle Deprecations and map on to new configuration // FIXME: DEPRECATED to be removed in v1.19.0 deprecatedSetting("mailer", "MAILER_TYPE", "mailer", "PROTOCOL") if sec.HasKey("MAILER_TYPE") && !sec.HasKey("PROTOCOL") { if sec.Key("MAILER_TYPE").String() == "sendmail" { - MailService.Protocol = "sendmail" + sec.Key("PROTOCOL").MustString("sendmail") } } @@ -96,31 +74,91 @@ func newMailService() { if err != nil { log.Fatal("Invalid mailer.HOST (%s): %v", givenHost, err) } - MailService.SMTPAddr = addr - MailService.SMTPPort = port + sec.Key("SMTP_ADDR").MustString(addr) + sec.Key("SMTP_PORT").MustString(port) } // FIXME: DEPRECATED to be removed in v1.19.0 deprecatedSetting("mailer", "IS_TLS_ENABLED", "mailer", "PROTOCOL") if sec.HasKey("IS_TLS_ENABLED") && !sec.HasKey("PROTOCOL") { if sec.Key("IS_TLS_ENABLED").MustBool() { - MailService.Protocol = "smtps" + sec.Key("PROTOCOL").MustString("smtps") } else { - MailService.Protocol = "smtp+startls" + sec.Key("PROTOCOL").MustString("smtp+starttls") } } + // FIXME: DEPRECATED to be removed in v1.19.0 + deprecatedSetting("mailer", "DISABLE_HELO", "mailer", "ENABLE_HELO") + if sec.HasKey("DISABLE_HELO") && !sec.HasKey("ENABLE_HELO") { + sec.Key("ENABLE_HELO").MustBool(!sec.Key("DISABLE_HELO").MustBool()) + } + + // FIXME: DEPRECATED to be removed in v1.19.0 + deprecatedSetting("mailer", "SKIP_VERIFY", "mailer", "FORCE_TRUST_SERVER_CERT") + if sec.HasKey("SKIP_VERIFY") && !sec.HasKey("FORCE_TRUST_SERVER_CERT") { + sec.Key("FORCE_TRUST_SERVER_CERT").MustBool(sec.Key("SKIP_VERIFY").MustBool()) + } + + // FIXME: DEPRECATED to be removed in v1.19.0 + deprecatedSetting("mailer", "USE_CERTIFICATE", "mailer", "USE_CLIENT_CERT") + if sec.HasKey("USE_CERTIFICATE") && !sec.HasKey("USE_CLIENT_CERT") { + sec.Key("USE_CLIENT_CERT").MustBool(sec.Key("USE_CERTIFICATE").MustBool()) + } + + // FIXME: DEPRECATED to be removed in v1.19.0 + deprecatedSetting("mailer", "CERT_FILE", "mailer", "CLIENT_CERT_FILE") + if sec.HasKey("CERT_FILE") && !sec.HasKey("CLIENT_CERT_FILE") { + sec.Key("CERT_FILE").MustString(sec.Key("CERT_FILE").String()) + } + + // FIXME: DEPRECATED to be removed in v1.19.0 + deprecatedSetting("mailer", "KEY_FILE", "mailer", "CLIENT_KEY_FILE") + if sec.HasKey("KEY_FILE") && !sec.HasKey("CLIENT_KEY_FILE") { + sec.Key("KEY_FILE").MustString(sec.Key("KEY_FILE").String()) + } + + // FIXME: DEPRECATED to be removed in v1.19.0 + deprecatedSetting("mailer", "ENABLE_HTML_ALTERNATIVE", "mailer", "SEND_AS_PLAIN_TEXT") + if sec.HasKey("ENABLE_HTML_ALTERNATIVE") && !sec.HasKey("SEND_AS_PLAIN_TEXT") { + sec.Key("SEND_AS_PLAIN_TEXT").MustBool(!sec.Key("ENABLE_HTML_ALTERNATIVE").MustBool(false)) + } + + if sec.HasKey("PROTOCOL") && sec.Key("PROTOCOL").String() == "smtp+startls" { + log.Error("Deprecated fallback `[mailer]` `PROTOCOL = smtp+startls` present. Use `[mailer]` `PROTOCOL = smtp+starttls`` instead. This fallback will be removed in v1.19.0") + sec.Key("PROTOCOL").SetValue("smtp+starttls") + } + + // Set default values & validate + sec.Key("NAME").MustString(AppName) + sec.Key("PROTOCOL").In("", []string{"smtp", "smtps", "smtp+starttls", "smtp+unix", "sendmail", "dummy"}) + sec.Key("ENABLE_HELO").MustBool(true) + sec.Key("FORCE_TRUST_SERVER_CERT").MustBool(false) + sec.Key("USE_CLIENT_CERT").MustBool(false) + sec.Key("SENDMAIL_PATH").MustString("sendmail") + sec.Key("SENDMAIL_TIMEOUT").MustDuration(5 * time.Minute) + sec.Key("SENDMAIL_CONVERT_CRLF").MustBool(true) + sec.Key("FROM").MustString(sec.Key("USER").String()) + + // Now map the values on to the MailService + MailService = &Mailer{} + if err := sec.MapTo(MailService); err != nil { + log.Fatal("Unable to map [mailer] section on to MailService. Error: %v", err) + } + + // Infer SMTPPort if not set if MailService.SMTPPort == "" { switch MailService.Protocol { case "smtp": MailService.SMTPPort = "25" case "smtps": MailService.SMTPPort = "465" - case "smtp+startls": + case "smtp+starttls": MailService.SMTPPort = "587" } } + // Infer Protocol if MailService.Protocol == "" { if strings.ContainsAny(MailService.SMTPAddr, "/\\") { MailService.Protocol = "smtp+unix" @@ -131,7 +169,7 @@ func newMailService() { case "465": MailService.Protocol = "smtps" case "587": - MailService.Protocol = "smtp+startls" + MailService.Protocol = "smtp+starttls" default: log.Error("unable to infer unspecified mailer.PROTOCOL from mailer.SMTP_PORT = %q, assume using smtps", MailService.SMTPPort) MailService.Protocol = "smtps" @@ -151,42 +189,6 @@ func newMailService() { } } - // FIXME: DEPRECATED to be removed in v1.19.0 - deprecatedSetting("mailer", "DISABLE_HELO", "mailer", "ENABLE_HELO") - if sec.HasKey("DISABLE_HELO") && !sec.HasKey("ENABLE_HELO") { - MailService.EnableHelo = !sec.Key("DISABLE_HELO").MustBool() - } - - // FIXME: DEPRECATED to be removed in v1.19.0 - deprecatedSetting("mailer", "SKIP_VERIFY", "mailer", "FORCE_TRUST_SERVER_CERT") - if sec.HasKey("SKIP_VERIFY") && !sec.HasKey("FORCE_TRUST_SERVER_CERT") { - MailService.ForceTrustServerCert = sec.Key("SKIP_VERIFY").MustBool() - } - - // FIXME: DEPRECATED to be removed in v1.19.0 - deprecatedSetting("mailer", "USE_CERTIFICATE", "mailer", "USE_CLIENT_CERT") - if sec.HasKey("USE_CERTIFICATE") && !sec.HasKey("USE_CLIENT_CERT") { - MailService.UseClientCert = sec.Key("USE_CLIENT_CERT").MustBool() - } - - // FIXME: DEPRECATED to be removed in v1.19.0 - deprecatedSetting("mailer", "CERT_FILE", "mailer", "CLIENT_CERT_FILE") - if sec.HasKey("CERT_FILE") && !sec.HasKey("CLIENT_CERT_FILE") { - MailService.ClientCertFile = sec.Key("CERT_FILE").String() - } - - // FIXME: DEPRECATED to be removed in v1.19.0 - deprecatedSetting("mailer", "KEY_FILE", "mailer", "CLIENT_KEY_FILE") - if sec.HasKey("KEY_FILE") && !sec.HasKey("CLIENT_KEY_FILE") { - MailService.ClientKeyFile = sec.Key("KEY_FILE").String() - } - - // FIXME: DEPRECATED to be removed in v1.19.0 - deprecatedSetting("mailer", "ENABLE_HTML_ALTERNATIVE", "mailer", "SEND_AS_PLAIN_TEXT") - if sec.HasKey("ENABLE_HTML_ALTERNATIVE") && !sec.HasKey("SEND_AS_PLAIN_TEXT") { - MailService.SendAsPlainText = !sec.Key("ENABLE_HTML_ALTERNATIVE").MustBool(false) - } - if MailService.From != "" { parsed, err := mail.ParseAddress(MailService.From) if err != nil { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index bd035403c8..bb2281f13b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -603,7 +603,7 @@ func LoadForTest(extraConfigs ...string) { func deprecatedSetting(oldSection, oldKey, newSection, newKey string) { if Cfg.Section(oldSection).HasKey(oldKey) { - log.Error("Deprecated fallback `[%s]` `%s` present. Use `[%s]` `%s` instead. This fallback will be removed in v1.18.0", oldSection, oldKey, newSection, newKey) + log.Error("Deprecated fallback `[%s]` `%s` present. Use `[%s]` `%s` instead. This fallback will be removed in v1.19.0", oldSection, oldKey, newSection, newKey) } } diff --git a/services/mailer/mailer.go b/services/mailer/mailer.go index 46b0c8e2f4..2663b6b2ba 100644 --- a/services/mailer/mailer.go +++ b/services/mailer/mailer.go @@ -166,7 +166,7 @@ func (s *smtpSender) Send(from string, to []string, msg io.WriterTo) error { defer conn.Close() var tlsconfig *tls.Config - if opts.Protocol == "smtps" || opts.Protocol == "smtp+startls" { + if opts.Protocol == "smtps" || opts.Protocol == "smtp+starttls" { tlsconfig = &tls.Config{ InsecureSkipVerify: opts.ForceTrustServerCert, ServerName: opts.SMTPAddr, @@ -208,7 +208,7 @@ func (s *smtpSender) Send(from string, to []string, msg io.WriterTo) error { } } - if opts.Protocol == "smtp+startls" { + if opts.Protocol == "smtp+starttls" { hasStartTLS, _ := client.Extension("STARTTLS") if hasStartTLS { if err = client.StartTLS(tlsconfig); err != nil { From da956b863b0f623fccb47b3168c4560cbe1282ab Mon Sep 17 00:00:00 2001 From: silverwind Date: Fri, 2 Dec 2022 21:42:41 +0100 Subject: [PATCH 036/182] Multiple improvements for comment edit diff (#21990) (#22007) Backport #21990 - Use explicit avatar size so when JS copies the HTML, the size gets copied with it - Replace icon font use with SVG - Improve styling and diff rendering - Sort lists in `svg.js` Fixes: https://github.com/go-gitea/gitea/issues/21924 Co-authored-by: Lunny Xiao Co-authored-by: techknowlogick --- routers/web/repo/issue_content_history.go | 13 +++++++---- web_src/js/features/repo-issue-content.js | 21 ++++++++--------- web_src/js/svg.js | 28 ++++++++++++----------- web_src/less/_repository.less | 18 +++++++++++++++ 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/routers/web/repo/issue_content_history.go b/routers/web/repo/issue_content_history.go index a9386d274a..7d188df4a0 100644 --- a/routers/web/repo/issue_content_history.go +++ b/routers/web/repo/issue_content_history.go @@ -6,16 +6,17 @@ package repo import ( "bytes" - "fmt" "html" "net/http" "strings" + "code.gitea.io/gitea/models/avatars" issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/timeutil" "github.com/sergi/go-diff/diffmatchpatch" @@ -64,16 +65,20 @@ func GetContentHistoryList(ctx *context.Context) { } else { actionText = ctx.Locale.Tr("repo.issues.content_history.edited") } - timeSinceText := timeutil.TimeSinceUnix(item.EditedUnix, ctx.Locale) username := item.UserName if setting.UI.DefaultShowFullName && strings.TrimSpace(item.UserFullName) != "" { username = strings.TrimSpace(item.UserFullName) } + src := html.EscapeString(item.UserAvatarLink) + class := avatars.DefaultAvatarClass + " mr-3" + name := html.EscapeString(username) + avatarHTML := string(templates.AvatarHTML(src, 28, class, username)) + timeSinceText := string(timeutil.TimeSinceUnix(item.EditedUnix, ctx.Locale)) + results = append(results, map[string]interface{}{ - "name": fmt.Sprintf("%s %s %s", - html.EscapeString(item.UserAvatarLink), html.EscapeString(username), actionText, timeSinceText), + "name": avatarHTML + "" + name + " " + actionText + " " + timeSinceText, "value": item.HistoryID, }) } diff --git a/web_src/js/features/repo-issue-content.js b/web_src/js/features/repo-issue-content.js index a671200d88..37801d2ad4 100644 --- a/web_src/js/features/repo-issue-content.js +++ b/web_src/js/features/repo-issue-content.js @@ -13,20 +13,17 @@ function showContentHistoryDetail(issueBaseUrl, commentId, historyId, itemTitleH $dialog = $(`
- -
- ${itemTitleHtml} - `); $dialog.appendTo($('body')); $dialog.find('.dialog-header-options').dropdown({ @@ -62,10 +59,10 @@ function showContentHistoryDetail(issueBaseUrl, commentId, historyId, itemTitleH _csrf: csrfToken, }, }).done((resp) => { - $dialog.find('.content').html(resp.diffHtml); + $dialog.find('.comment-diff-data').removeClass('is-loading').html(resp.diffHtml); // there is only one option "item[data-option-item=delete]", so the dropdown can be entirely shown/hidden. if (resp.canSoftDelete) { - $dialog.find('.dialog-header-options').show(); + $dialog.find('.dialog-header-options').removeClass('hide'); } }); }, @@ -79,7 +76,7 @@ function showContentHistoryMenu(issueBaseUrl, $item, commentId) { const $headerLeft = $item.find('.comment-header-left'); const menuHtml = `
- • ${i18nTextEdited} ${svg('octicon-triangle-down', 17)} + • ${i18nTextEdited}${svg('octicon-triangle-down', 14, 'dropdown icon ml-1 mt-1')}
`; diff --git a/web_src/js/svg.js b/web_src/js/svg.js index dedc126303..60dd49f8bf 100644 --- a/web_src/js/svg.js +++ b/web_src/js/svg.js @@ -1,11 +1,12 @@ import octiconChevronDown from '../../public/img/svg/octicon-chevron-down.svg'; import octiconChevronRight from '../../public/img/svg/octicon-chevron-right.svg'; -import octiconCopy from '../../public/img/svg/octicon-copy.svg'; import octiconClock from '../../public/img/svg/octicon-clock.svg'; +import octiconCopy from '../../public/img/svg/octicon-copy.svg'; import octiconDiffAdded from '../../public/img/svg/octicon-diff-added.svg'; import octiconDiffModified from '../../public/img/svg/octicon-diff-modified.svg'; import octiconDiffRemoved from '../../public/img/svg/octicon-diff-removed.svg'; import octiconDiffRenamed from '../../public/img/svg/octicon-diff-renamed.svg'; +import octiconFile from '../../public/img/svg/octicon-file.svg'; import octiconFileDirectoryFill from '../../public/img/svg/octicon-file-directory-fill.svg'; import octiconGitMerge from '../../public/img/svg/octicon-git-merge.svg'; import octiconGitPullRequest from '../../public/img/svg/octicon-git-pull-request.svg'; @@ -20,17 +21,23 @@ import octiconProject from '../../public/img/svg/octicon-project.svg'; import octiconRepo from '../../public/img/svg/octicon-repo.svg'; import octiconRepoForked from '../../public/img/svg/octicon-repo-forked.svg'; import octiconRepoTemplate from '../../public/img/svg/octicon-repo-template.svg'; -import octiconTriangleDown from '../../public/img/svg/octicon-triangle-down.svg'; -import octiconFile from '../../public/img/svg/octicon-file.svg'; -import octiconSidebarExpand from '../../public/img/svg/octicon-sidebar-expand.svg'; import octiconSidebarCollapse from '../../public/img/svg/octicon-sidebar-collapse.svg'; +import octiconSidebarExpand from '../../public/img/svg/octicon-sidebar-expand.svg'; +import octiconTriangleDown from '../../public/img/svg/octicon-triangle-down.svg'; +import octiconX from '../../public/img/svg/octicon-x.svg'; export const svgs = { 'octicon-chevron-down': octiconChevronDown, 'octicon-chevron-right': octiconChevronRight, - 'octicon-copy': octiconCopy, 'octicon-clock': octiconClock, + 'octicon-copy': octiconCopy, + 'octicon-diff-added': octiconDiffAdded, + 'octicon-diff-modified': octiconDiffModified, + 'octicon-diff-removed': octiconDiffRemoved, + 'octicon-diff-renamed': octiconDiffRenamed, + 'octicon-file': octiconFile, + 'octicon-file-directory-fill': octiconFileDirectoryFill, 'octicon-git-merge': octiconGitMerge, 'octicon-git-pull-request': octiconGitPullRequest, 'octicon-issue-closed': octiconIssueClosed, @@ -44,15 +51,10 @@ export const svgs = { 'octicon-repo': octiconRepo, 'octicon-repo-forked': octiconRepoForked, 'octicon-repo-template': octiconRepoTemplate, - 'octicon-triangle-down': octiconTriangleDown, - 'octicon-file': octiconFile, - 'octicon-file-directory-fill': octiconFileDirectoryFill, - 'octicon-sidebar-expand': octiconSidebarExpand, 'octicon-sidebar-collapse': octiconSidebarCollapse, - 'octicon-diff-added': octiconDiffAdded, - 'octicon-diff-modified': octiconDiffModified, - 'octicon-diff-removed': octiconDiffRemoved, - 'octicon-diff-renamed': octiconDiffRenamed, + 'octicon-sidebar-expand': octiconSidebarExpand, + 'octicon-triangle-down': octiconTriangleDown, + 'octicon-x': octiconX, }; diff --git a/web_src/less/_repository.less b/web_src/less/_repository.less index 38103c5f8e..20775e1cf7 100644 --- a/web_src/less/_repository.less +++ b/web_src/less/_repository.less @@ -2907,6 +2907,24 @@ tbody.commit-list { text-align: left; } +.comment-diff-data { + background: var(--color-code-bg); + max-height: calc(100vh - 10.5rem); + overflow-y: auto; +} + +.comment-diff-data pre { + line-height: 18px; + white-space: pre-wrap; + word-break: break-all; + overflow-wrap: break-word; +} + +.content-history-detail-dialog .header .avatar { + position: relative; + top: -2px; +} + #topic_edit { margin-top: 5px; } From c7770fa50213b979ba0a3cd20f9008268e398140 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 4 Dec 2022 05:58:58 +0000 Subject: [PATCH 037/182] Use path not filepath in template filenames (#21993) (#22022) Backport #21993 Paths in git are always separated by `/` not `\` - therefore we should `path` and not `filepath` Fix #21987 Signed-off-by: Andrew Thornton Co-authored-by: Lunny Xiao Co-authored-by: Lauris BH --- modules/issue/template/unmarshal.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/issue/template/unmarshal.go b/modules/issue/template/unmarshal.go index 3398719cf6..9b684f1bf7 100644 --- a/modules/issue/template/unmarshal.go +++ b/modules/issue/template/unmarshal.go @@ -7,7 +7,7 @@ package template import ( "fmt" "io" - "path/filepath" + "path" "strconv" "code.gitea.io/gitea/modules/git" @@ -44,7 +44,7 @@ func Unmarshal(filename string, content []byte) (*api.IssueTemplate, error) { // UnmarshalFromEntry parses out a valid template from the blob in entry func UnmarshalFromEntry(entry *git.TreeEntry, dir string) (*api.IssueTemplate, error) { - return unmarshalFromEntry(entry, filepath.Join(dir, entry.Name())) + return unmarshalFromEntry(entry, path.Join(dir, entry.Name())) // Filepaths in Git are ALWAYS '/' separated do not use filepath here } // UnmarshalFromCommit parses out a valid template from the commit @@ -109,7 +109,7 @@ func unmarshal(filename string, content []byte) (*api.IssueTemplate, error) { // It could be a valid markdown with two horizontal lines, or an invalid markdown with wrong metadata. it.Content = string(content) - it.Name = filepath.Base(it.FileName) + it.Name = path.Base(it.FileName) // paths in Git are always '/' separated - do not use filepath! it.About, _ = util.SplitStringAtByteN(it.Content, 80) } else { it.Content = templateBody From 9819a477176fa6a9c037da3f7b7d9e7b6c2b456f Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 4 Dec 2022 11:29:19 +0100 Subject: [PATCH 038/182] On tag/branch-exist check, dont panic if repo is nil (#21787) (#21788) backport #21787 --- modules/git/repo_branch_nogogit.go | 2 +- modules/git/repo_tag_nogogit.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index 95c3718841..26e20ba116 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -53,7 +53,7 @@ func (repo *Repository) IsReferenceExist(name string) bool { // IsBranchExist returns true if given branch exists in current repository. func (repo *Repository) IsBranchExist(name string) bool { - if name == "" { + if repo == nil || name == "" { return false } diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index 5d3aace52f..0bb0da21bf 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -16,7 +16,7 @@ import ( // IsTagExist returns true if given tag exists in the repository. func (repo *Repository) IsTagExist(name string) bool { - if name == "" { + if repo == nil || name == "" { return false } From 757b49ec5e677cf32782761a4cd777c4d0dd16e0 Mon Sep 17 00:00:00 2001 From: zeripath Date: Sun, 4 Dec 2022 11:10:54 +0000 Subject: [PATCH 039/182] Do not emit ambiguous character warning on rendered pages (#22016) (#22018) Backport #22016 The real sensitivity of ambiguous characters is in source code - therefore warning about them in rendered pages causes too many warnings. Therefore simply remove the warning on rendered pages. The escape button will remain available and it is present on the view source page. Fix #20999 Signed-off-by: Andrew Thornton --- templates/repo/view_file.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index 2b4f279cb6..ed2162a414 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -79,7 +79,9 @@
- {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}} + {{if not (or .IsMarkup .IsRenderedHTML)}} + {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}} + {{end}}
{{if .IsMarkup}} {{if .FileContent}}{{.FileContent | Safe}}{{end}} From 09794b42598a8426d34ecbb034b80cb9398543cd Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 5 Dec 2022 06:13:11 +0800 Subject: [PATCH 040/182] Fix ListBranches to handle empty case (#21921) (#22024) Fix #21910 Backport #21921 Co-authored-by: KN4CK3R Co-authored-by: KN4CK3R --- routers/api/v1/repo/branch.go | 64 ++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go index 84a172e92b..b0ef5bc96e 100644 --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -252,42 +252,50 @@ func ListBranches(ctx *context.APIContext) { // "200": // "$ref": "#/responses/BranchList" - listOptions := utils.GetListOptions(ctx) - skip, _ := listOptions.GetStartEnd() - branches, totalNumOfBranches, err := ctx.Repo.GitRepo.GetBranches(skip, listOptions.PageSize) - if err != nil { - ctx.Error(http.StatusInternalServerError, "GetBranches", err) - return - } + var totalNumOfBranches int + var apiBranches []*api.Branch - apiBranches := make([]*api.Branch, 0, len(branches)) - for i := range branches { - c, err := branches[i].GetCommit() + listOptions := utils.GetListOptions(ctx) + + if !ctx.Repo.Repository.IsEmpty && ctx.Repo.GitRepo != nil { + skip, _ := listOptions.GetStartEnd() + branches, total, err := ctx.Repo.GitRepo.GetBranches(skip, listOptions.PageSize) if err != nil { - // Skip if this branch doesn't exist anymore. - if git.IsErrNotExist(err) { - totalNumOfBranches-- - continue + ctx.Error(http.StatusInternalServerError, "GetBranches", err) + return + } + + apiBranches = make([]*api.Branch, 0, len(branches)) + for i := range branches { + c, err := branches[i].GetCommit() + if err != nil { + // Skip if this branch doesn't exist anymore. + if git.IsErrNotExist(err) { + total-- + continue + } + ctx.Error(http.StatusInternalServerError, "GetCommit", err) + return } - ctx.Error(http.StatusInternalServerError, "GetCommit", err) - return + branchProtection, err := git_model.GetProtectedBranchBy(ctx, ctx.Repo.Repository.ID, branches[i].Name) + if err != nil { + ctx.Error(http.StatusInternalServerError, "GetProtectedBranchBy", err) + return + } + apiBranch, err := convert.ToBranch(ctx.Repo.Repository, branches[i], c, branchProtection, ctx.Doer, ctx.Repo.IsAdmin()) + if err != nil { + ctx.Error(http.StatusInternalServerError, "convert.ToBranch", err) + return + } + apiBranches = append(apiBranches, apiBranch) } - branchProtection, err := git_model.GetProtectedBranchBy(ctx, ctx.Repo.Repository.ID, branches[i].Name) - if err != nil { - ctx.Error(http.StatusInternalServerError, "GetBranchProtection", err) - return - } - apiBranch, err := convert.ToBranch(ctx.Repo.Repository, branches[i], c, branchProtection, ctx.Doer, ctx.Repo.IsAdmin()) - if err != nil { - ctx.Error(http.StatusInternalServerError, "convert.ToBranch", err) - return - } - apiBranches = append(apiBranches, apiBranch) + + totalNumOfBranches = total } ctx.SetLinkHeader(totalNumOfBranches, listOptions.PageSize) ctx.SetTotalCountHeader(int64(totalNumOfBranches)) - ctx.JSON(http.StatusOK, &apiBranches) + ctx.JSON(http.StatusOK, apiBranches) } // GetBranchProtection gets a branch protection From 6e4ba048430b364968f8861731575483cbeb011e Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 5 Dec 2022 09:20:38 +0000 Subject: [PATCH 041/182] Ensure that Chinese punctuation is not ambiguous when locale is Chinese (#22019) (#22030) Backport #22019 Although there are per-locale fallbacks for ambiguity the locale names for Chinese do not quite match our locales. This PR simply maps zh-CN on to zh-hans and other zh variants on to zh-hant. Ref #20999 Signed-off-by: Andrew Thornton Co-authored-by: Lauris BH --- modules/charset/ambiguous.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/charset/ambiguous.go b/modules/charset/ambiguous.go index 9dab3b0951..ef5e7650a6 100644 --- a/modules/charset/ambiguous.go +++ b/modules/charset/ambiguous.go @@ -29,6 +29,12 @@ func AmbiguousTablesForLocale(locale translation.Locale) []*AmbiguousTable { key = key[:idx] } } + if table == nil && (locale.Language() == "zh-CN" || locale.Language() == "zh_CN") { + table = AmbiguousCharacters["zh-hans"] + } + if table == nil && strings.HasPrefix(locale.Language(), "zh") { + table = AmbiguousCharacters["zh-hant"] + } if table == nil { table = AmbiguousCharacters["_default"] } From 35fc9ad9847fc17df11cac14be7fbef8125eba48 Mon Sep 17 00:00:00 2001 From: zeripath Date: Mon, 5 Dec 2022 12:20:37 +0000 Subject: [PATCH 042/182] Use GhostUser if needed for TrackedTimes (#22021) (#22029) Backport #22021 When getting tracked times out of the db and loading their attributes handle not exist errors in a nicer way. (Also prevent an NPE.) Fix #22006 Signed-off-by: Andrew Thornton Signed-off-by: Andrew Thornton Co-authored-by: Lunny Xiao --- models/issues/tracked_time.go | 52 +++++++++++++++++++++-------------- modules/convert/issue.go | 11 ++++---- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/models/issues/tracked_time.go b/models/issues/tracked_time.go index ca21eb5149..f2412881ec 100644 --- a/models/issues/tracked_time.go +++ b/models/issues/tracked_time.go @@ -6,6 +6,7 @@ package issues import ( "context" + "errors" "time" "code.gitea.io/gitea/models/db" @@ -47,33 +48,42 @@ func (t *TrackedTime) LoadAttributes() (err error) { } func (t *TrackedTime) loadAttributes(ctx context.Context) (err error) { + // Load the issue if t.Issue == nil { t.Issue, err = GetIssueByID(ctx, t.IssueID) - if err != nil { - return - } - err = t.Issue.LoadRepo(ctx) - if err != nil { - return - } - } - if t.User == nil { - t.User, err = user_model.GetUserByIDCtx(ctx, t.UserID) - if err != nil { - return - } - } - return err -} -// LoadAttributes load Issue, User -func (tl TrackedTimeList) LoadAttributes() (err error) { - for _, t := range tl { - if err = t.LoadAttributes(); err != nil { + if err != nil && !errors.Is(err, util.ErrNotExist) { return err } } - return err + // Now load the repo for the issue (which we may have just loaded) + if t.Issue != nil { + err = t.Issue.LoadRepo(ctx) + if err != nil && !errors.Is(err, util.ErrNotExist) { + return err + } + } + // Load the user + if t.User == nil { + t.User, err = user_model.GetUserByIDCtx(ctx, t.UserID) + if err != nil { + if !errors.Is(err, util.ErrNotExist) { + return err + } + t.User = user_model.NewGhostUser() + } + } + return nil +} + +// LoadAttributes load Issue, User +func (tl TrackedTimeList) LoadAttributes() error { + for _, t := range tl { + if err := t.LoadAttributes(); err != nil { + return err + } + } + return nil } // FindTrackedTimesOptions represent the filters for tracked times. If an ID is 0 it will be ignored. diff --git a/modules/convert/issue.go b/modules/convert/issue.go index 5364367a80..feedc6f8de 100644 --- a/modules/convert/issue.go +++ b/modules/convert/issue.go @@ -110,12 +110,11 @@ func ToAPIIssueList(il issues_model.IssueList) []*api.Issue { // ToTrackedTime converts TrackedTime to API format func ToTrackedTime(t *issues_model.TrackedTime) (apiT *api.TrackedTime) { apiT = &api.TrackedTime{ - ID: t.ID, - IssueID: t.IssueID, - UserID: t.UserID, - UserName: t.User.Name, - Time: t.Time, - Created: t.Created, + ID: t.ID, + IssueID: t.IssueID, + UserID: t.UserID, + Time: t.Time, + Created: t.Created, } if t.Issue != nil { apiT.Issue = ToAPIIssue(t.Issue) From ac543315497b70c4156ad8cc710134a7d696e552 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Tue, 6 Dec 2022 12:01:58 +0100 Subject: [PATCH 043/182] Add dumb-init to rootless docker (#22046) - backport #21775 --- Dockerfile.rootless | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.rootless b/Dockerfile.rootless index 8c2b8e98c9..1368c0cce7 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -31,6 +31,7 @@ EXPOSE 2222 3000 RUN apk --no-cache add \ bash \ ca-certificates \ + dumb-init \ gettext \ git \ curl \ @@ -68,6 +69,6 @@ ENV HOME "/var/lib/gitea/git" VOLUME ["/var/lib/gitea", "/etc/gitea"] WORKDIR /var/lib/gitea -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +ENTRYPOINT ["/usr/bin/dumb-init", "--", "/usr/local/bin/docker-entrypoint.sh"] CMD [] From e39bb2d05aba9eecfa7e93ca42245be2166a741a Mon Sep 17 00:00:00 2001 From: zeripath Date: Tue, 6 Dec 2022 13:16:47 +0000 Subject: [PATCH 044/182] Handle empty author names (#21902) (#22027) Backport #21902 Although git does expect that author names should be of the form: `NAME ` some users have been able to create commits with: `` Fix #21900 Signed-off-by: Andrew Thornton Co-authored-by: Lauris BH Co-authored-by: Lunny Xiao --- modules/git/signature_gogit.go | 5 ++++- modules/git/signature_nogogit.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/git/signature_gogit.go b/modules/git/signature_gogit.go index 6f1c98420d..5ab38cd852 100644 --- a/modules/git/signature_gogit.go +++ b/modules/git/signature_gogit.go @@ -10,6 +10,7 @@ package git import ( "bytes" "strconv" + "strings" "time" "github.com/go-git/go-git/v5/plumbing/object" @@ -30,7 +31,9 @@ type Signature = object.Signature func newSignatureFromCommitline(line []byte) (_ *Signature, err error) { sig := new(Signature) emailStart := bytes.IndexByte(line, '<') - sig.Name = string(line[:emailStart-1]) + if emailStart > 0 { // Empty name has already occurred, even if it shouldn't + sig.Name = strings.TrimSpace(string(line[:emailStart-1])) + } emailEnd := bytes.IndexByte(line, '>') sig.Email = string(line[emailStart+1 : emailEnd]) diff --git a/modules/git/signature_nogogit.go b/modules/git/signature_nogogit.go index 07a3b79f1e..3fa5c8da3e 100644 --- a/modules/git/signature_nogogit.go +++ b/modules/git/signature_nogogit.go @@ -11,6 +11,7 @@ import ( "bytes" "fmt" "strconv" + "strings" "time" ) @@ -51,7 +52,9 @@ func newSignatureFromCommitline(line []byte) (sig *Signature, err error) { return } - sig.Name = string(line[:emailStart-1]) + if emailStart > 0 { // Empty name has already occurred, even if it shouldn't + sig.Name = strings.TrimSpace(string(line[:emailStart-1])) + } sig.Email = string(line[emailStart+1 : emailEnd]) hasTime := emailEnd+2 < len(line) From c0ca9c612b70c14f41d6d9a0f915ece899bd0188 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 7 Dec 2022 14:20:12 +0800 Subject: [PATCH 045/182] Fix issue/PR numbers (#22037) (#22044) Backport #22037. When deleting a closed issue, we should update both `NumIssues`and `NumClosedIssues`, or `NumOpenIssues`(`= NumIssues -NumClosedIssues`) will be wrong. It's the same for pull requests. Releated to #21557. Alse fixed two harmless problems: - The SQL to check issue/PR total numbers is wrong, that means it will update the numbers even if they are correct. - Replace legacy `num_issues = num_issues + 1` operations with `UpdateRepoIssueNumbers`. Co-authored-by: Lunny Xiao --- models/issues/issue.go | 7 +------ models/repo.go | 4 ++-- services/issue/issue.go | 9 ++++++++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/models/issues/issue.go b/models/issues/issue.go index ca48f425f2..b9aa649d21 100644 --- a/models/issues/issue.go +++ b/models/issues/issue.go @@ -1010,12 +1010,7 @@ func NewIssueWithIndex(ctx context.Context, doer *user_model.User, opts NewIssue } } - if opts.IsPull { - _, err = e.Exec("UPDATE `repository` SET num_pulls = num_pulls + 1 WHERE id = ?", opts.Issue.RepoID) - } else { - _, err = e.Exec("UPDATE `repository` SET num_issues = num_issues + 1 WHERE id = ?", opts.Issue.RepoID) - } - if err != nil { + if err := repo_model.UpdateRepoIssueNumbers(ctx, opts.Issue.RepoID, opts.IsPull, false); err != nil { return err } diff --git a/models/repo.go b/models/repo.go index 569dafee5f..26fe51b5a6 100644 --- a/models/repo.go +++ b/models/repo.go @@ -444,7 +444,7 @@ func CheckRepoStats(ctx context.Context) error { }, // Repository.NumIssues { - statsQuery("SELECT repo.id FROM `repository` repo WHERE repo.num_issues!=(SELECT COUNT(*) FROM `issue` WHERE repo_id=repo.id AND is_closed=? AND is_pull=?)", false, false), + statsQuery("SELECT repo.id FROM `repository` repo WHERE repo.num_issues!=(SELECT COUNT(*) FROM `issue` WHERE repo_id=repo.id AND is_pull=?)", false), repoStatsCorrectNumIssues, "repository count 'num_issues'", }, @@ -456,7 +456,7 @@ func CheckRepoStats(ctx context.Context) error { }, // Repository.NumPulls { - statsQuery("SELECT repo.id FROM `repository` repo WHERE repo.num_pulls!=(SELECT COUNT(*) FROM `issue` WHERE repo_id=repo.id AND is_closed=? AND is_pull=?)", false, true), + statsQuery("SELECT repo.id FROM `repository` repo WHERE repo.num_pulls!=(SELECT COUNT(*) FROM `issue` WHERE repo_id=repo.id AND is_pull=?)", true), repoStatsCorrectNumPulls, "repository count 'num_pulls'", }, diff --git a/services/issue/issue.go b/services/issue/issue.go index 47782e50d3..42548d9d9d 100644 --- a/services/issue/issue.go +++ b/services/issue/issue.go @@ -220,9 +220,16 @@ func deleteIssue(issue *issues_model.Issue) error { return err } - if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, issue.IsClosed); err != nil { + // update the total issue numbers + if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, false); err != nil { return err } + // if the issue is closed, update the closed issue numbers + if issue.IsClosed { + if err := repo_model.UpdateRepoIssueNumbers(ctx, issue.RepoID, issue.IsPull, true); err != nil { + return err + } + } if err := issues_model.UpdateMilestoneCounters(ctx, issue.MilestoneID); err != nil { return fmt.Errorf("error updating counters for milestone id %d: %w", From b54c064f89725b803bcc2a2017eec6895ad07175 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 10 Dec 2022 01:11:46 +0100 Subject: [PATCH 046/182] Workaround for container registry push/pull errors (#21862) (#22068) Backport of #21862 Co-authored-by: Lunny Xiao --- modules/packages/content_store.go | 7 ++++ routers/api/packages/container/blob.go | 32 ++++++++++++++++++- routers/api/packages/container/container.go | 28 ++++++++++++++-- routers/api/packages/container/manifest.go | 12 +++++++ .../api_packages_container_test.go | 28 ++++++++++++++++ 5 files changed, 103 insertions(+), 4 deletions(-) diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index a3a5d1a666..b0e653a102 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -30,6 +30,13 @@ func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { return s.store.Open(KeyToRelativePath(key)) } +// FIXME: Workaround to be removed in v1.20 +// https://github.com/go-gitea/gitea/issues/19586 +func (s *ContentStore) Has(key BlobHash256Key) error { + _, err := s.store.Stat(KeyToRelativePath(key)) + return err +} + // Save stores a package blob func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error { _, err := s.store.Save(KeyToRelativePath(key), r, size) diff --git a/routers/api/packages/container/blob.go b/routers/api/packages/container/blob.go index 8a9cbd4a15..7494fd639b 100644 --- a/routers/api/packages/container/blob.go +++ b/routers/api/packages/container/blob.go @@ -7,8 +7,11 @@ package container import ( "context" "encoding/hex" + "errors" "fmt" + "os" "strings" + "sync" "code.gitea.io/gitea/models/db" packages_model "code.gitea.io/gitea/models/packages" @@ -16,9 +19,12 @@ import ( "code.gitea.io/gitea/modules/log" packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" + "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" ) +var uploadVersionMutex sync.Mutex + // saveAsPackageBlob creates a package blob from an upload // The uploaded blob gets stored in a special upload version to link them to the package/image func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_service.PackageInfo) (*packages_model.PackageBlob, error) { @@ -28,6 +34,11 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic contentStore := packages_module.NewContentStore() + var uploadVersion *packages_model.PackageVersion + + // FIXME: Replace usage of mutex with database transaction + // https://github.com/go-gitea/gitea/pull/21862 + uploadVersionMutex.Lock() err := db.WithTx(func(ctx context.Context) error { created := true p := &packages_model.Package{ @@ -68,11 +79,30 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic } } + uploadVersion = pv + + return nil + }) + uploadVersionMutex.Unlock() + if err != nil { + return nil, err + } + + err = db.WithTx(func(ctx context.Context) error { pb, exists, err = packages_model.GetOrInsertBlob(ctx, pb) if err != nil { log.Error("Error inserting package blob: %v", err) return err } + // FIXME: Workaround to be removed in v1.20 + // https://github.com/go-gitea/gitea/issues/19586 + if exists { + err = contentStore.Has(packages_module.BlobHash256Key(pb.HashSHA256)) + if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256) + exists = false + } + } if !exists { if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), hsr, hsr.Size()); err != nil { log.Error("Error saving package blob in content store: %v", err) @@ -83,7 +113,7 @@ func saveAsPackageBlob(hsr packages_module.HashedSizeReader, pi *packages_servic filename := strings.ToLower(fmt.Sprintf("sha256_%s", pb.HashSHA256)) pf := &packages_model.PackageFile{ - VersionID: pv.ID, + VersionID: uploadVersion.ID, BlobID: pb.ID, Name: filename, LowerName: filename, diff --git a/routers/api/packages/container/container.go b/routers/api/packages/container/container.go index 5bc64e1b29..7af06a9171 100644 --- a/routers/api/packages/container/container.go +++ b/routers/api/packages/container/container.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/url" + "os" "regexp" "strconv" "strings" @@ -24,6 +25,7 @@ import ( container_module "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/packages/container/oci" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/api/packages/helper" packages_service "code.gitea.io/gitea/services/packages" container_service "code.gitea.io/gitea/services/packages/container" @@ -193,7 +195,7 @@ func InitiateUploadBlob(ctx *context.Context) { mount := ctx.FormTrim("mount") from := ctx.FormTrim("from") if mount != "" { - blob, _ := container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + blob, _ := workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ Image: from, Digest: mount, }) @@ -406,7 +408,7 @@ func getBlobFromContext(ctx *context.Context) (*packages_model.PackageFileDescri return nil, container_model.ErrContainerBlobNotExist } - return container_model.GetContainerBlob(ctx, &container_model.BlobSearchOptions{ + return workaroundGetContainerBlob(ctx, &container_model.BlobSearchOptions{ OwnerID: ctx.Package.Owner.ID, Image: ctx.Params("image"), Digest: digest, @@ -548,7 +550,7 @@ func getManifestFromContext(ctx *context.Context) (*packages_model.PackageFileDe return nil, container_model.ErrContainerBlobNotExist } - return container_model.GetContainerBlob(ctx, opts) + return workaroundGetContainerBlob(ctx, opts) } // https://github.com/opencontainers/distribution-spec/blob/main/spec.md#checking-if-content-exists-in-the-registry @@ -688,3 +690,23 @@ func GetTagList(ctx *context.Context) { Tags: tags, }) } + +// FIXME: Workaround to be removed in v1.20 +// https://github.com/go-gitea/gitea/issues/19586 +func workaroundGetContainerBlob(ctx *context.Context, opts *container_model.BlobSearchOptions) (*packages_model.PackageFileDescriptor, error) { + blob, err := container_model.GetContainerBlob(ctx, opts) + if err != nil { + return nil, err + } + + err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(blob.Blob.HashSHA256)) + if err != nil { + if errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", blob.Blob.HashSHA256) + return nil, container_model.ErrContainerBlobNotExist + } + return nil, err + } + + return blob, nil +} diff --git a/routers/api/packages/container/manifest.go b/routers/api/packages/container/manifest.go index 8beed3dbb7..f7225ec395 100644 --- a/routers/api/packages/container/manifest.go +++ b/routers/api/packages/container/manifest.go @@ -6,8 +6,10 @@ package container import ( "context" + "errors" "fmt" "io" + "os" "strings" "code.gitea.io/gitea/models/db" @@ -19,6 +21,7 @@ import ( packages_module "code.gitea.io/gitea/modules/packages" container_module "code.gitea.io/gitea/modules/packages/container" "code.gitea.io/gitea/modules/packages/container/oci" + "code.gitea.io/gitea/modules/util" packages_service "code.gitea.io/gitea/services/packages" ) @@ -403,6 +406,15 @@ func createManifestBlob(ctx context.Context, mci *manifestCreationInfo, pv *pack log.Error("Error inserting package blob: %v", err) return nil, false, "", err } + // FIXME: Workaround to be removed in v1.20 + // https://github.com/go-gitea/gitea/issues/19586 + if exists { + err = packages_module.NewContentStore().Has(packages_module.BlobHash256Key(pb.HashSHA256)) + if err != nil && (errors.Is(err, util.ErrNotExist) || errors.Is(err, os.ErrNotExist)) { + log.Debug("Package registry inconsistent: blob %s does not exist on file system", pb.HashSHA256) + exists = false + } + } if !exists { contentStore := packages_module.NewContentStore() if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), buf, buf.Size()); err != nil { diff --git a/tests/integration/api_packages_container_test.go b/tests/integration/api_packages_container_test.go index ba76ee4baa..60cbecd067 100644 --- a/tests/integration/api_packages_container_test.go +++ b/tests/integration/api_packages_container_test.go @@ -6,10 +6,12 @@ package integration import ( "bytes" + "crypto/sha256" "encoding/base64" "fmt" "net/http" "strings" + "sync" "testing" "code.gitea.io/gitea/models/db" @@ -594,6 +596,32 @@ func TestPackageContainer(t *testing.T) { }) } + // https://github.com/go-gitea/gitea/issues/19586 + t.Run("ParallelUpload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + url := fmt.Sprintf("%sv2/%s/parallel", setting.AppURL, user.Name) + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + + content := []byte{byte(i)} + digest := fmt.Sprintf("sha256:%x", sha256.Sum256(content)) + + go func() { + defer wg.Done() + + req := NewRequestWithBody(t, "POST", fmt.Sprintf("%s/blobs/uploads?digest=%s", url, digest), bytes.NewReader(content)) + addTokenAuthHeader(req, userToken) + resp := MakeRequest(t, req, http.StatusCreated) + + assert.Equal(t, digest, resp.Header().Get("Docker-Content-Digest")) + }() + } + wg.Wait() + }) + t.Run("OwnerNameChange", func(t *testing.T) { defer tests.PrintCurrentTest(t)() From 079ef568247f9216a6e66c7e6038e11cc2bdd5f5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 13 Dec 2022 03:59:50 +0800 Subject: [PATCH 047/182] Fix permission check on issue/pull lock (#22113) backport #22110 --- routers/web/web.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/routers/web/web.go b/routers/web/web.go index 9e0440d09a..8ab90f7eda 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -626,7 +626,6 @@ func RegisterRoutes(m *web.Route) { reqRepoReleaseWriter := context.RequireRepoWriter(unit.TypeReleases) reqRepoReleaseReader := context.RequireRepoReader(unit.TypeReleases) reqRepoWikiWriter := context.RequireRepoWriter(unit.TypeWiki) - reqRepoIssueWriter := context.RequireRepoWriter(unit.TypeIssues) reqRepoIssueReader := context.RequireRepoReader(unit.TypeIssues) reqRepoPullsReader := context.RequireRepoReader(unit.TypePullRequests) reqRepoIssuesOrPullsWriter := context.RequireRepoWriterOr(unit.TypeIssues, unit.TypePullRequests) @@ -947,8 +946,8 @@ func RegisterRoutes(m *web.Route) { }) }) m.Post("/reactions/{action}", bindIgnErr(forms.ReactionForm{}), repo.ChangeIssueReaction) - m.Post("/lock", reqRepoIssueWriter, bindIgnErr(forms.IssueLockForm{}), repo.LockIssue) - m.Post("/unlock", reqRepoIssueWriter, repo.UnlockIssue) + m.Post("/lock", reqRepoIssuesOrPullsWriter, bindIgnErr(forms.IssueLockForm{}), repo.LockIssue) + m.Post("/unlock", reqRepoIssuesOrPullsWriter, repo.UnlockIssue) m.Post("/delete", reqRepoAdmin, repo.DeleteIssue) }, context.RepoMustNotBeArchived()) m.Group("/{index}", func() { From c36a1bc766effd6cafd860548d6a911d503144c1 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 13 Dec 2022 18:59:18 +0800 Subject: [PATCH 048/182] Fix parallel creating commit status bug with tests (#21911) (#21989) backport #21911 backport #21998 Co-authored-by: silverwind --- models/db/index.go | 25 ++++-- models/git/commit_status.go | 120 ++++++++++++------------- tests/integration/repo_commits_test.go | 31 +++++++ 3 files changed, 106 insertions(+), 70 deletions(-) diff --git a/models/db/index.go b/models/db/index.go index 58a976ad52..f64bf6bfb5 100644 --- a/models/db/index.go +++ b/models/db/index.go @@ -8,6 +8,9 @@ import ( "context" "errors" "fmt" + "strconv" + + "code.gitea.io/gitea/modules/setting" ) // ResourceIndex represents a resource index which could be used as issue/release and others @@ -24,11 +27,6 @@ var ( ErrGetResourceIndexFailed = errors.New("get resource index failed") ) -const ( - // MaxDupIndexAttempts max retry times to create index - MaxDupIndexAttempts = 3 -) - // SyncMaxResourceIndex sync the max index with the resource func SyncMaxResourceIndex(ctx context.Context, tableName string, groupID, maxIndex int64) (err error) { e := GetEngine(ctx) @@ -61,8 +59,25 @@ func SyncMaxResourceIndex(ctx context.Context, tableName string, groupID, maxInd return nil } +func postgresGetNextResourceIndex(ctx context.Context, tableName string, groupID int64) (int64, error) { + res, err := GetEngine(ctx).Query(fmt.Sprintf("INSERT INTO %s (group_id, max_index) "+ + "VALUES (?,1) ON CONFLICT (group_id) DO UPDATE SET max_index = %s.max_index+1 RETURNING max_index", + tableName, tableName), groupID) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, ErrGetResourceIndexFailed + } + return strconv.ParseInt(string(res[0]["max_index"]), 10, 64) +} + // GetNextResourceIndex generates a resource index, it must run in the same transaction where the resource is created func GetNextResourceIndex(ctx context.Context, tableName string, groupID int64) (int64, error) { + if setting.Database.UsePostgreSQL { + return postgresGetNextResourceIndex(ctx, tableName, groupID) + } + e := GetEngine(ctx) // try to update the max_index to next value, and acquire the write-lock for the record diff --git a/models/git/commit_status.go b/models/git/commit_status.go index 620baa036c..53c5458040 100644 --- a/models/git/commit_status.go +++ b/models/git/commit_status.go @@ -7,8 +7,10 @@ package git import ( "context" "crypto/sha1" + "errors" "fmt" "net/url" + "strconv" "strings" "time" @@ -49,79 +51,67 @@ func init() { db.RegisterModel(new(CommitStatusIndex)) } -// upsertCommitStatusIndex the function will not return until it acquires the lock or receives an error. -func upsertCommitStatusIndex(ctx context.Context, repoID int64, sha string) (err error) { - // An atomic UPSERT operation (INSERT/UPDATE) is the only operation - // that ensures that the key is actually locked. - switch { - case setting.Database.UseSQLite3 || setting.Database.UsePostgreSQL: - _, err = db.Exec(ctx, "INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+ - "VALUES (?,?,1) ON CONFLICT (repo_id,sha) DO UPDATE SET max_index = `commit_status_index`.max_index+1", - repoID, sha) - case setting.Database.UseMySQL: - _, err = db.Exec(ctx, "INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+ - "VALUES (?,?,1) ON DUPLICATE KEY UPDATE max_index = max_index+1", - repoID, sha) - case setting.Database.UseMSSQL: - // https://weblogs.sqlteam.com/dang/2009/01/31/upsert-race-condition-with-merge/ - _, err = db.Exec(ctx, "MERGE `commit_status_index` WITH (HOLDLOCK) as target "+ - "USING (SELECT ? AS repo_id, ? AS sha) AS src "+ - "ON src.repo_id = target.repo_id AND src.sha = target.sha "+ - "WHEN MATCHED THEN UPDATE SET target.max_index = target.max_index+1 "+ - "WHEN NOT MATCHED THEN INSERT (repo_id, sha, max_index) "+ - "VALUES (src.repo_id, src.sha, 1);", - repoID, sha) - default: - return fmt.Errorf("database type not supported") +func postgresGetCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) { + res, err := db.GetEngine(ctx).Query("INSERT INTO `commit_status_index` (repo_id, sha, max_index) "+ + "VALUES (?,?,1) ON CONFLICT (repo_id, sha) DO UPDATE SET max_index = `commit_status_index`.max_index+1 RETURNING max_index", + repoID, sha) + if err != nil { + return 0, err } - return err + if len(res) == 0 { + return 0, db.ErrGetResourceIndexFailed + } + return strconv.ParseInt(string(res[0]["max_index"]), 10, 64) } // GetNextCommitStatusIndex retried 3 times to generate a resource index -func GetNextCommitStatusIndex(repoID int64, sha string) (int64, error) { - for i := 0; i < db.MaxDupIndexAttempts; i++ { - idx, err := getNextCommitStatusIndex(repoID, sha) - if err == db.ErrResouceOutdated { - continue - } +func GetNextCommitStatusIndex(ctx context.Context, repoID int64, sha string) (int64, error) { + if setting.Database.UsePostgreSQL { + return postgresGetCommitStatusIndex(ctx, repoID, sha) + } + + e := db.GetEngine(ctx) + + // try to update the max_index to next value, and acquire the write-lock for the record + res, err := e.Exec("UPDATE `commit_status_index` SET max_index=max_index+1 WHERE repo_id=? AND sha=?", repoID, sha) + if err != nil { + return 0, err + } + affected, err := res.RowsAffected() + if err != nil { + return 0, err + } + if affected == 0 { + // this slow path is only for the first time of creating a resource index + _, errIns := e.Exec("INSERT INTO `commit_status_index` (repo_id, sha, max_index) VALUES (?, ?, 0)", repoID, sha) + res, err = e.Exec("UPDATE `commit_status_index` SET max_index=max_index+1 WHERE repo_id=? AND sha=?", repoID, sha) if err != nil { return 0, err } - return idx, nil - } - return 0, db.ErrGetResourceIndexFailed -} -// getNextCommitStatusIndex return the next index -func getNextCommitStatusIndex(repoID int64, sha string) (int64, error) { - ctx, commiter, err := db.TxContext() - if err != nil { - return 0, err - } - defer commiter.Close() - - var preIdx int64 - _, err = db.GetEngine(ctx).SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id = ? AND sha = ?", repoID, sha).Get(&preIdx) - if err != nil { - return 0, err + affected, err = res.RowsAffected() + if err != nil { + return 0, err + } + // if the update still can not update any records, the record must not exist and there must be some errors (insert error) + if affected == 0 { + if errIns == nil { + return 0, errors.New("impossible error when GetNextCommitStatusIndex, insert and update both succeeded but no record is updated") + } + return 0, errIns + } } - if err := upsertCommitStatusIndex(ctx, repoID, sha); err != nil { - return 0, err - } - - var curIdx int64 - has, err := db.GetEngine(ctx).SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id = ? AND sha = ? AND max_index=?", repoID, sha, preIdx+1).Get(&curIdx) + // now, the new index is in database (protected by the transaction and write-lock) + var newIdx int64 + has, err := e.SQL("SELECT max_index FROM `commit_status_index` WHERE repo_id=? AND sha=?", repoID, sha).Get(&newIdx) if err != nil { return 0, err } if !has { - return 0, db.ErrResouceOutdated + return 0, errors.New("impossible error when GetNextCommitStatusIndex, upsert succeeded but no record can be selected") } - if err := commiter.Commit(); err != nil { - return 0, err - } - return curIdx, nil + return newIdx, nil } func (status *CommitStatus) loadAttributes(ctx context.Context) (err error) { @@ -291,18 +281,18 @@ func NewCommitStatus(opts NewCommitStatusOptions) error { return fmt.Errorf("NewCommitStatus[%s, %s]: no user specified", repoPath, opts.SHA) } - // Get the next Status Index - idx, err := GetNextCommitStatusIndex(opts.Repo.ID, opts.SHA) - if err != nil { - return fmt.Errorf("generate commit status index failed: %w", err) - } - ctx, committer, err := db.TxContext() if err != nil { return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", opts.Repo.ID, opts.Creator.ID, opts.SHA, err) } defer committer.Close() + // Get the next Status Index + idx, err := GetNextCommitStatusIndex(ctx, opts.Repo.ID, opts.SHA) + if err != nil { + return fmt.Errorf("generate commit status index failed: %w", err) + } + opts.CommitStatus.Description = strings.TrimSpace(opts.CommitStatus.Description) opts.CommitStatus.Context = strings.TrimSpace(opts.CommitStatus.Context) opts.CommitStatus.TargetURL = strings.TrimSpace(opts.CommitStatus.TargetURL) @@ -316,7 +306,7 @@ func NewCommitStatus(opts NewCommitStatusOptions) error { // Insert new CommitStatus if _, err = db.GetEngine(ctx).Insert(opts.CommitStatus); err != nil { - return fmt.Errorf("Insert CommitStatus[%s, %s]: %w", repoPath, opts.SHA, err) + return fmt.Errorf("insert CommitStatus[%s, %s]: %w", repoPath, opts.SHA, err) } return committer.Commit() diff --git a/tests/integration/repo_commits_test.go b/tests/integration/repo_commits_test.go index c9e7753596..95dbfbae5f 100644 --- a/tests/integration/repo_commits_test.go +++ b/tests/integration/repo_commits_test.go @@ -5,9 +5,11 @@ package integration import ( + "fmt" "net/http" "net/http/httptest" "path" + "sync" "testing" "code.gitea.io/gitea/modules/json" @@ -115,3 +117,32 @@ func TestRepoCommitsWithStatusFailure(t *testing.T) { func TestRepoCommitsWithStatusWarning(t *testing.T) { doTestRepoCommitWithStatus(t, "warning", "gitea-exclamation", "yellow") } + +func TestRepoCommitsStatusParallel(t *testing.T) { + defer tests.PrepareTestEnv(t)() + + session := loginUser(t, "user2") + + // Request repository commits page + req := NewRequest(t, "GET", "/user2/repo1/commits/branch/master") + resp := session.MakeRequest(t, req, http.StatusOK) + + doc := NewHTMLParser(t, resp.Body) + // Get first commit URL + commitURL, exists := doc.doc.Find("#commits-table tbody tr td.sha a").Attr("href") + assert.True(t, exists) + assert.NotEmpty(t, commitURL) + + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + wg.Add(1) + go func(parentT *testing.T, i int) { + parentT.Run(fmt.Sprintf("ParallelCreateStatus_%d", i), func(t *testing.T) { + runBody := doAPICreateCommitStatus(NewAPITestContext(t, "user2", "repo1"), path.Base(commitURL), api.CommitStatusState("pending")) + runBody(t) + wg.Done() + }) + }(t, i) + } + wg.Wait() +} From 1409b348c6594aba5b51de8ec8dfa84752cf09c5 Mon Sep 17 00:00:00 2001 From: aceArt-GmbH <33117017+aceArt-GmbH@users.noreply.github.com> Date: Tue, 13 Dec 2022 23:37:33 +0100 Subject: [PATCH 049/182] Fix sorting admin user list by last login (#22081) (#22107) --- routers/web/explore/user.go | 4 ++++ templates/admin/user/list.tmpl | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/routers/web/explore/user.go b/routers/web/explore/user.go index ea3d83e8d6..260cfc4efa 100644 --- a/routers/web/explore/user.go +++ b/routers/web/explore/user.go @@ -69,6 +69,10 @@ func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions, orderBy = "`user`.updated_unix ASC" case "reversealphabetically": orderBy = "`user`.name DESC" + case "lastlogin": + orderBy = "`user`.last_login_unix ASC" + case "reverselastlogin": + orderBy = "`user`.last_login_unix DESC" case UserSearchDefaultSortType: // "alphabetically" default: orderBy = "`user`.name ASC" diff --git a/templates/admin/user/list.tmpl b/templates/admin/user/list.tmpl index 56f6eaa3ad..67a726eb3f 100644 --- a/templates/admin/user/list.tmpl +++ b/templates/admin/user/list.tmpl @@ -76,9 +76,9 @@ {{.locale.Tr "admin.users.2fa"}} {{.locale.Tr "admin.users.repos"}} {{.locale.Tr "admin.users.created"}} - + {{.locale.Tr "admin.users.last_login"}} - {{SortArrow "leastupdate" "recentupdate" $.SortType false}} + {{SortArrow "lastlogin" "reverselastlogin" $.SortType false}} {{.locale.Tr "admin.users.edit"}} From 194b780cd7f943d5155faf026223f4d7ff764daa Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 14 Dec 2022 13:13:16 +0000 Subject: [PATCH 050/182] Make gitea work using cmd.exe again (#22073) (#22133) Backport #22073 Gitea will attempt to lookup its location using LookPath however, this fails on cmd.exe if gitea is in the current working directory. exec.LookPath will return an exec.ErrDot error which we can test for and then simply using filepath.Abs(os.Args[0]) to absolute gitea against the current working directory. Fix #22063 Signed-off-by: Andrew Thornton --- modules/setting/setting.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index bb2281f13b..5fe4682968 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -464,6 +464,13 @@ func getAppPath() (string, error) { appPath, err = exec.LookPath(os.Args[0]) } + if err != nil { + // FIXME: Once we switch to go 1.19 use !errors.Is(err, exec.ErrDot) + if !strings.Contains(err.Error(), "cannot run executable found relative to current directory") { + return "", err + } + appPath, err = filepath.Abs(os.Args[0]) + } if err != nil { return "", err } From 9e49270676f4c38e5bff3679ddbca6a92f6325e0 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 14 Dec 2022 19:38:15 +0100 Subject: [PATCH 051/182] Fix condition for is_internal (#22095) (#22132) Backport of #22095 I changed it to a static condition because it needs a new version of xorm which is only available in 1.19. This change is valid because `SearchLatestVersions` is never called to list internal versions and there will no change to this behaviour in <1.19. Co-authored-by: Lunny Xiao Co-authored-by: techknowlogick --- models/packages/package_version.go | 2 +- routers/api/packages/npm/npm.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/models/packages/package_version.go b/models/packages/package_version.go index f9965bcb74..4b2a6f84c3 100644 --- a/models/packages/package_version.go +++ b/models/packages/package_version.go @@ -305,7 +305,7 @@ func SearchLatestVersions(ctx context.Context, opts *PackageSearchOptions) ([]*P sess := db.GetEngine(ctx). Table("package_version"). - Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))"). + Join("LEFT", "package_version pv2", "package_version.package_id = pv2.package_id AND pv2.is_internal = ? AND (package_version.created_unix < pv2.created_unix OR (package_version.created_unix = pv2.created_unix AND package_version.id < pv2.id))", false). Join("INNER", "package", "package.id = package_version.package_id"). Where(cond) diff --git a/routers/api/packages/npm/npm.go b/routers/api/packages/npm/npm.go index a9676b9191..502d353ffc 100644 --- a/routers/api/packages/npm/npm.go +++ b/routers/api/packages/npm/npm.go @@ -402,8 +402,9 @@ func setPackageTag(tag string, pv *packages_model.PackageVersion, deleteOnly boo func PackageSearch(ctx *context.Context) { pvs, total, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{ - OwnerID: ctx.Package.Owner.ID, - Type: packages_model.TypeNpm, + OwnerID: ctx.Package.Owner.ID, + Type: packages_model.TypeNpm, + IsInternal: util.OptionalBoolFalse, Name: packages_model.SearchValue{ ExactMatch: false, Value: ctx.FormTrim("text"), From dd2343d01fff8851009545aac93e712549183655 Mon Sep 17 00:00:00 2001 From: zeripath Date: Thu, 15 Dec 2022 00:59:16 +0000 Subject: [PATCH 052/182] Correctly handle moved files in apply patch (#22118) (#22135) Backport #22118 Moved files in a patch will result in git apply returning: ``` error: {filename}: No such file or directory ``` This wasn't handled by the git apply patch code. This PR adds handling for this. Fix #22083 Signed-off-by: Andrew Thornton Co-authored-by: KN4CK3R --- services/pull/patch.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/pull/patch.go b/services/pull/patch.go index 9b87ac22e2..d3e049fc1b 100644 --- a/services/pull/patch.go +++ b/services/pull/patch.go @@ -53,6 +53,8 @@ var patchErrorSuffices = []string{ ": patch does not apply", ": already exists in working directory", "unrecognized input", + ": No such file or directory", + ": does not exist in index", } // TestPatch will test whether a simple patch will apply @@ -416,6 +418,7 @@ func checkConflicts(ctx context.Context, pr *issues_model.PullRequest, gitRepo * scanner := bufio.NewScanner(stderrReader) for scanner.Scan() { line := scanner.Text() + log.Trace("PullRequest[%d].testPatch: stderr: %s", pr.ID, line) if strings.HasPrefix(line, prefix) { conflict = true filepath := strings.TrimSpace(strings.Split(line[len(prefix):], ":")[0]) From 4297aced9360c3ce6f10764ea7f16286e66319bc Mon Sep 17 00:00:00 2001 From: silverwind Date: Fri, 16 Dec 2022 20:57:22 +0100 Subject: [PATCH 053/182] Fix margin and alignment in dashboard repolist (#22120) (#22122) Backport #22120 to 1.18. Seems this has recently regressed, previously, there was a significant whitespace between icon and text, but it seems to be gone, so I added the margin. --- templates/user/dashboard/repolist.tmpl | 6 +++--- web_src/less/_dashboard.less | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/templates/user/dashboard/repolist.tmpl b/templates/user/dashboard/repolist.tmpl index 620cc322f0..9d0ea01ec5 100644 --- a/templates/user/dashboard/repolist.tmpl +++ b/templates/user/dashboard/repolist.tmpl @@ -128,9 +128,9 @@