refactor: decouple context from migration structs (#33399)

Use context as much as possible.

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
TheFox0x7 2025-02-07 06:37:32 +01:00 committed by GitHub
parent 466cc725bc
commit 1ec8d80fa3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 455 additions and 509 deletions

View file

@ -64,7 +64,6 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// from github via APIv3
type GithubDownloaderV3 struct {
base.NullDownloader
ctx context.Context
clients []*github.Client
baseURL string
repoOwner string
@ -79,12 +78,11 @@ type GithubDownloaderV3 struct {
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
func NewGithubDownloaderV3(_ context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
downloader := GithubDownloaderV3{
userName: userName,
baseURL: baseURL,
password: password,
ctx: ctx,
repoOwner: repoOwner,
repoName: repoName,
maxPerPage: 100,
@ -141,12 +139,7 @@ func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
g.rates = append(g.rates, nil)
}
// SetContext set context
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
g.ctx = ctx
}
func (g *GithubDownloaderV3) waitAndPickClient() {
func (g *GithubDownloaderV3) waitAndPickClient(ctx context.Context) {
var recentIdx int
var maxRemaining int
for i := 0; i < len(g.clients); i++ {
@ -160,13 +153,13 @@ func (g *GithubDownloaderV3) waitAndPickClient() {
for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
select {
case <-g.ctx.Done():
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
err := g.RefreshRate()
err := g.RefreshRate(ctx)
if err != nil {
log.Error("g.getClient().RateLimit.Get: %s", err)
}
@ -174,8 +167,8 @@ func (g *GithubDownloaderV3) waitAndPickClient() {
}
// RefreshRate update the current rate (doesn't count in rate limit)
func (g *GithubDownloaderV3) RefreshRate() error {
rates, _, err := g.getClient().RateLimit.Get(g.ctx)
func (g *GithubDownloaderV3) RefreshRate(ctx context.Context) error {
rates, _, err := g.getClient().RateLimit.Get(ctx)
if err != nil {
// if rate limit is not enabled, ignore it
if strings.Contains(err.Error(), "404") {
@ -198,9 +191,9 @@ func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
}
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
g.waitAndPickClient()
gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
func (g *GithubDownloaderV3) GetRepoInfo(ctx context.Context) (*base.Repository, error) {
g.waitAndPickClient(ctx)
gr, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
@ -219,9 +212,9 @@ func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
}
// GetTopics return github topics
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
g.waitAndPickClient()
r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
func (g *GithubDownloaderV3) GetTopics(ctx context.Context) ([]string, error) {
g.waitAndPickClient(ctx)
r, resp, err := g.getClient().Repositories.Get(ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
@ -230,12 +223,12 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
}
// GetMilestones returns milestones
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
func (g *GithubDownloaderV3) GetMilestones(ctx context.Context) ([]*base.Milestone, error) {
perPage := g.maxPerPage
milestones := make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
g.waitAndPickClient()
ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
g.waitAndPickClient(ctx)
ms, resp, err := g.getClient().Issues.ListMilestones(ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
@ -279,12 +272,12 @@ func convertGithubLabel(label *github.Label) *base.Label {
}
// GetLabels returns labels
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
func (g *GithubDownloaderV3) GetLabels(ctx context.Context) ([]*base.Label, error) {
perPage := g.maxPerPage
labels := make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
g.waitAndPickClient()
ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
g.waitAndPickClient(ctx)
ls, resp, err := g.getClient().Issues.ListLabels(ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@ -304,7 +297,7 @@ func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
return labels, nil
}
func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
func (g *GithubDownloaderV3) convertGithubRelease(ctx context.Context, rel *github.RepositoryRelease) *base.Release {
// GitHub allows commitish to be a reference.
// In this case, we need to remove the prefix, i.e. convert "refs/heads/main" to "main".
targetCommitish := strings.TrimPrefix(rel.GetTargetCommitish(), git.BranchPrefix)
@ -339,12 +332,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time,
DownloadFunc: func() (io.ReadCloser, error) {
g.waitAndPickClient()
readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
g.waitAndPickClient(ctx)
readCloser, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(ctx, g.repoOwner, g.repoName, assetID, nil)
if err != nil {
return nil, err
}
if err := g.RefreshRate(); err != nil {
if err := g.RefreshRate(ctx); err != nil {
log.Error("g.getClient().RateLimits: %s", err)
}
@ -364,13 +357,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
return io.NopCloser(strings.NewReader(redirectURL)), nil
}
g.waitAndPickClient()
req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
g.waitAndPickClient(ctx)
req, err := http.NewRequestWithContext(ctx, "GET", redirectURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
err1 := g.RefreshRate()
err1 := g.RefreshRate(ctx)
if err1 != nil {
log.Error("g.RefreshRate(): %s", err1)
}
@ -385,12 +378,12 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
}
// GetReleases returns releases
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
func (g *GithubDownloaderV3) GetReleases(ctx context.Context) ([]*base.Release, error) {
perPage := g.maxPerPage
releases := make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
g.waitAndPickClient()
ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
g.waitAndPickClient(ctx)
ls, resp, err := g.getClient().Repositories.ListReleases(ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
@ -401,7 +394,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
g.setRate(&resp.Rate)
for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release))
releases = append(releases, g.convertGithubRelease(ctx, release))
}
if len(ls) < perPage {
break
@ -411,7 +404,7 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
func (g *GithubDownloaderV3) GetIssues(ctx context.Context, page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
@ -426,8 +419,8 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
}
allIssues := make([]*base.Issue, 0, perPage)
g.waitAndPickClient()
issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
g.waitAndPickClient(ctx)
issues, resp, err := g.getClient().Issues.ListByRepo(ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err)
}
@ -447,8 +440,8 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
@ -503,12 +496,12 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
}
// GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) {
comments, err := g.getComments(commentable)
func (g *GithubDownloaderV3) GetComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, bool, error) {
comments, err := g.getComments(ctx, commentable)
return comments, false, err
}
func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) {
func (g *GithubDownloaderV3) getComments(ctx context.Context, commentable base.Commentable) ([]*base.Comment, error) {
var (
allComments = make([]*base.Comment, 0, g.maxPerPage)
created = "created"
@ -522,8 +515,8 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
},
}
for {
g.waitAndPickClient()
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt)
g.waitAndPickClient(ctx)
comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, int(commentable.GetForeignIndex()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}
@ -533,8 +526,8 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
@ -576,7 +569,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
}
// GetAllComments returns repository comments according page and perPageSize
func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) {
func (g *GithubDownloaderV3) GetAllComments(ctx context.Context, page, perPage int) ([]*base.Comment, bool, error) {
var (
allComments = make([]*base.Comment, 0, perPage)
created = "created"
@ -594,8 +587,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
},
}
g.waitAndPickClient()
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
g.waitAndPickClient(ctx)
comments, resp, err := g.getClient().Issues.ListComments(ctx, g.repoOwner, g.repoName, 0, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err)
}
@ -608,8 +601,8 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
@ -648,7 +641,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
}
// GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
func (g *GithubDownloaderV3) GetPullRequests(ctx context.Context, page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
@ -662,8 +655,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
},
}
allPRs := make([]*base.PullRequest, 0, perPage)
g.waitAndPickClient()
prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
g.waitAndPickClient(ctx)
prs, resp, err := g.getClient().PullRequests.List(ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %w", err)
}
@ -679,8 +672,8 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListIssueReactions(ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
@ -702,7 +695,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
}
// download patch and saved as tmp file
g.waitAndPickClient()
g.waitAndPickClient(ctx)
allPRs = append(allPRs, &base.PullRequest{
Title: pr.GetTitle(),
@ -759,15 +752,15 @@ func convertGithubReview(r *github.PullRequestReview) *base.Review {
}
}
func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
func (g *GithubDownloaderV3) convertGithubReviewComments(ctx context.Context, cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
rcs := make([]*base.ReviewComment, 0, len(cs))
for _, c := range cs {
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
g.waitAndPickClient(ctx)
res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
@ -806,7 +799,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
}
// GetReviews returns pull requests review
func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Review, error) {
func (g *GithubDownloaderV3) GetReviews(ctx context.Context, reviewable base.Reviewable) ([]*base.Review, error) {
allReviews := make([]*base.Review, 0, g.maxPerPage)
if g.SkipReviews {
return allReviews, nil
@ -816,8 +809,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
}
// Get approve/request change reviews
for {
g.waitAndPickClient()
reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
g.waitAndPickClient(ctx)
reviews, resp, err := g.getClient().PullRequests.ListReviews(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}
@ -830,14 +823,14 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
PerPage: g.maxPerPage,
}
for {
g.waitAndPickClient()
reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2)
g.waitAndPickClient(ctx)
reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), review.GetID(), opt2)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}
g.setRate(&resp.Rate)
cs, err := g.convertGithubReviewComments(reviewComments)
cs, err := g.convertGithubReviewComments(ctx, reviewComments)
if err != nil {
return nil, err
}
@ -856,8 +849,8 @@ func (g *GithubDownloaderV3) GetReviews(reviewable base.Reviewable) ([]*base.Rev
}
// Get requested reviews
for {
g.waitAndPickClient()
reviewers, resp, err := g.getClient().PullRequests.ListReviewers(g.ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
g.waitAndPickClient(ctx)
reviewers, resp, err := g.getClient().PullRequests.ListReviewers(ctx, g.repoOwner, g.repoName, int(reviewable.GetForeignIndex()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %w", err)
}