From 5e79b396548a463791ee917c8ac589e3f418d6d3 Mon Sep 17 00:00:00 2001 From: Gusted Date: Thu, 3 Apr 2025 06:47:37 +0000 Subject: [PATCH] fix(migrations): transfer PR flow information (#7421) - When migrating a pull requests from a Gitea or Forgejo instance, check if the pull request was created via the AGit flow and transfer that bit of information to the migrated pull request. - Expose this bit of information as the `flow` field for the pull request. - We have to do a horrible Go hack with Gitea's [go-sdk](gitea.com/gitea/go-sdk) to list all pull requests while being able to decode it to a struct that contains the new `Flow` field. The library does not allow you to do this out of the box, so we have to use `go:linkname` to access the private method that allows us to do this. This in turn means we have to do some boilerplate code that the library otherwise would do for us. The better option would be forking, but that would be a hassle of keeping the library in sync. - Resolves forgejo/forgejo#5848 - Unit test added. Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7421 Reviewed-by: Earl Warren Co-authored-by: Gusted Co-committed-by: Gusted --- modules/migration/pullrequest.go | 1 + modules/structs/pull.go | 3 +- services/convert/pull.go | 1 + services/migrations/gitea_downloader.go | 41 +++++++++++++++++- services/migrations/gitea_downloader_test.go | 43 +++++++++++++++++++ services/migrations/gitea_sdk_hack.go | 16 +++++++ services/migrations/gitea_uploader.go | 1 + services/migrations/main_test.go | 1 + ...t-test%2Fpulls%3Flimit=50&page=1&state=all | 8 ++++ .../GET_%2Fapi%2Fv1%2Fsettings%2Fapi | 7 +++ .../full_download/GET_%2Fapi%2Fv1%2Fversion | 7 +++ templates/swagger/v1_json.tmpl | 5 +++ 12 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 services/migrations/gitea_sdk_hack.go create mode 100644 services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Frepos%2FGusted%2Fagit-test%2Fpulls%3Flimit=50&page=1&state=all create mode 100644 services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fsettings%2Fapi create mode 100644 services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fversion diff --git a/modules/migration/pullrequest.go b/modules/migration/pullrequest.go index 933348d671..0861ab24f1 100644 --- a/modules/migration/pullrequest.go +++ b/modules/migration/pullrequest.go @@ -34,6 +34,7 @@ type PullRequest struct { Assignees []string IsLocked bool `yaml:"is_locked"` Reactions []*Reaction + Flow int64 ForeignIndex int64 Context DownloaderContext `yaml:"-"` EnsuredSafe bool `yaml:"ensured_safe"` diff --git a/modules/structs/pull.go b/modules/structs/pull.go index ab627666c9..1ce7550e19 100644 --- a/modules/structs/pull.go +++ b/modules/structs/pull.go @@ -57,7 +57,8 @@ type PullRequest struct { // swagger:strfmt date-time Closed *time.Time `json:"closed_at"` - PinOrder int `json:"pin_order"` + PinOrder int `json:"pin_order"` + Flow int64 `json:"flow"` } // PRBranchInfo information about a branch diff --git a/services/convert/pull.go b/services/convert/pull.go index 0cb4070595..ca965a0d18 100644 --- a/services/convert/pull.go +++ b/services/convert/pull.go @@ -95,6 +95,7 @@ func ToAPIPullRequest(ctx context.Context, pr *issues_model.PullRequest, doer *u RequestedReviewersTeams: []*api.Team{}, AllowMaintainerEdit: pr.AllowMaintainerEdit, + Flow: int64(pr.Flow), Base: &api.PRBranchInfo{ Name: pr.BaseBranch, diff --git a/services/migrations/gitea_downloader.go b/services/migrations/gitea_downloader.go index 272a7d379a..133cc5c928 100644 --- a/services/migrations/gitea_downloader.go +++ b/services/migrations/gitea_downloader.go @@ -504,6 +504,28 @@ func (g *GiteaDownloader) GetComments(commentable base.Commentable) ([]*base.Com return allComments, true, nil } +type ForgejoPullRequest struct { + gitea_sdk.PullRequest + Flow int64 `json:"flow"` +} + +// Extracted from https://gitea.com/gitea/go-sdk/src/commit/164e3358bc02213954fb4380b821bed80a14824d/gitea/pull.go#L347-L364 +func (g *GiteaDownloader) fixPullHeadSha(pr *ForgejoPullRequest) error { + if pr.Base != nil && pr.Base.Repository != nil && pr.Base.Repository.Owner != nil && pr.Head != nil && pr.Head.Ref != "" && pr.Head.Sha == "" { + owner := pr.Base.Repository.Owner.UserName + repo := pr.Base.Repository.Name + refs, _, err := g.client.GetRepoRefs(owner, repo, pr.Head.Ref) + if err != nil { + return err + } + if len(refs) == 0 { + return fmt.Errorf("unable to resolve PR ref %q", pr.Head.Ref) + } + pr.Head.Sha = refs[0].Object.SHA + } + return nil +} + // GetPullRequests returns pull requests according page and perPage func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { @@ -511,16 +533,30 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } allPRs := make([]*base.PullRequest, 0, perPage) - prs, _, err := g.client.ListRepoPullRequests(g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{ + prs := make([]*ForgejoPullRequest, 0, perPage) + opt := gitea_sdk.ListPullRequestsOptions{ ListOptions: gitea_sdk.ListOptions{ Page: page, PageSize: perPage, }, State: gitea_sdk.StateAll, - }) + } + + link, _ := url.Parse(fmt.Sprintf("/repos/%s/%s/pulls", url.PathEscape(g.repoOwner), url.PathEscape(g.repoName))) + link.RawQuery = opt.QueryEncode() + _, err := getParsedResponse(g.client, "GET", link.String(), http.Header{"content-type": []string{"application/json"}}, nil, &prs) if err != nil { return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err) } + + if g.client.CheckServerVersionConstraint(">= 1.14.0") != nil { + for i := range prs { + if err := g.fixPullHeadSha(prs[i]); err != nil { + return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %w", page, perPage, err) + } + } + } + for _, pr := range prs { var milestone string if pr.Milestone != nil { @@ -598,6 +634,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques MergeCommitSHA: mergeCommitSHA, IsLocked: pr.IsLocked, PatchURL: pr.PatchURL, + Flow: pr.Flow, Head: base.PullRequestBranch{ Ref: headRef, SHA: headSHA, diff --git a/services/migrations/gitea_downloader_test.go b/services/migrations/gitea_downloader_test.go index 2ca7098c9e..5acc3b86a9 100644 --- a/services/migrations/gitea_downloader_test.go +++ b/services/migrations/gitea_downloader_test.go @@ -307,3 +307,46 @@ func TestGiteaDownloadRepo(t *testing.T) { }, }, reviews) } + +func TestForgejoDownloadRepo(t *testing.T) { + token := os.Getenv("CODE_FORGEJO_TOKEN") + + fixturePath := "./testdata/code-forgejo-org/full_download" + server := unittest.NewMockWebServer(t, "https://code.forgejo.org", fixturePath, token != "") + defer server.Close() + + downloader, err := NewGiteaDownloader(t.Context(), server.URL, "Gusted/agit-test", "", "", token) + require.NoError(t, err) + require.NotNil(t, downloader) + + prs, _, err := downloader.GetPullRequests(1, 50) + require.NoError(t, err) + assert.Len(t, prs, 1) + + assertPullRequestEqual(t, &base.PullRequest{ + Number: 1, + PosterID: 63, + PosterName: "Gusted", + PosterEmail: "postmaster@gusted.xyz", + Title: "Add extra information", + State: "open", + Created: time.Date(2025, time.April, 1, 20, 28, 45, 0, time.UTC), + Updated: time.Date(2025, time.April, 1, 20, 28, 45, 0, time.UTC), + Base: base.PullRequestBranch{ + CloneURL: "", + Ref: "main", + SHA: "79ebb873a6497c8847141ba9706b3f757196a1e6", + RepoName: "agit-test", + OwnerName: "Gusted", + }, + Head: base.PullRequestBranch{ + CloneURL: server.URL + "/Gusted/agit-test.git", + Ref: "refs/pull/1/head", + SHA: "667e9317ec37b977e6d3d7d43e3440636970563c", + RepoName: "agit-test", + OwnerName: "Gusted", + }, + PatchURL: server.URL + "/Gusted/agit-test/pulls/1.patch", + Flow: 1, + }, prs[0]) +} diff --git a/services/migrations/gitea_sdk_hack.go b/services/migrations/gitea_sdk_hack.go new file mode 100644 index 0000000000..f3959717a8 --- /dev/null +++ b/services/migrations/gitea_sdk_hack.go @@ -0,0 +1,16 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: GPL-3.0-or-later + +package migrations + +import ( + "io" + "net/http" + + _ "unsafe" // Needed for go:linkname support + + gitea_sdk "code.gitea.io/sdk/gitea" +) + +//go:linkname getParsedResponse code.gitea.io/sdk/gitea.(*Client).getParsedResponse +func getParsedResponse(client *gitea_sdk.Client, method, path string, header http.Header, body io.Reader, obj any) (*gitea_sdk.Response, error) diff --git a/services/migrations/gitea_uploader.go b/services/migrations/gitea_uploader.go index 7edaef5557..55adad9685 100644 --- a/services/migrations/gitea_uploader.go +++ b/services/migrations/gitea_uploader.go @@ -802,6 +802,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*issues_model MergeBase: pr.Base.SHA, Index: pr.Number, HasMerged: pr.Merged, + Flow: issues_model.PullRequestFlow(pr.Flow), Issue: &issue, } diff --git a/services/migrations/main_test.go b/services/migrations/main_test.go index 17e6505df2..d543bd6d9c 100644 --- a/services/migrations/main_test.go +++ b/services/migrations/main_test.go @@ -136,6 +136,7 @@ func assertPullRequestEqual(t *testing.T, expected, actual *base.PullRequest) { assert.ElementsMatch(t, expected.Assignees, actual.Assignees) assert.Equal(t, expected.IsLocked, actual.IsLocked) assertReactionsEqual(t, expected.Reactions, actual.Reactions) + assert.Equal(t, expected.Flow, actual.Flow) } func assertPullRequestsEqual(t *testing.T, expected, actual []*base.PullRequest) { diff --git a/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Frepos%2FGusted%2Fagit-test%2Fpulls%3Flimit=50&page=1&state=all b/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Frepos%2FGusted%2Fagit-test%2Fpulls%3Flimit=50&page=1&state=all new file mode 100644 index 0000000000..87095d9e24 --- /dev/null +++ b/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Frepos%2FGusted%2Fagit-test%2Fpulls%3Flimit=50&page=1&state=all @@ -0,0 +1,8 @@ +Access-Control-Expose-Headers: X-Total-Count +Cache-Control: max-age=0, private, must-revalidate, no-transform +Content-Type: application/json;charset=utf-8 +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +X-Total-Count: 1 + +[{"id":4980,"url":"https://code.forgejo.org/Gusted/agit-test/pulls/1","number":1,"user":{"id":63,"login":"Gusted","login_name":"26734","source_id":1,"full_name":"","email":"postmaster@gusted.xyz","avatar_url":"https://code.forgejo.org/avatars/4ca5ad8bc488630869fdbd2051da61cbed7241c9c066d4e5e1dd36300f887340","html_url":"https://code.forgejo.org/Gusted","language":"en-US","is_admin":false,"last_login":"2025-04-01T16:35:18Z","created":"2023-07-08T13:33:38Z","restricted":false,"active":true,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":2,"following_count":0,"starred_repos_count":0,"username":"Gusted"},"title":"Add extra information","body":"","labels":[],"milestone":null,"assignee":null,"assignees":null,"requested_reviewers":[],"requested_reviewers_teams":[],"state":"open","draft":false,"is_locked":false,"comments":0,"review_comments":0,"additions":0,"deletions":0,"changed_files":0,"html_url":"https://code.forgejo.org/Gusted/agit-test/pulls/1","diff_url":"https://code.forgejo.org/Gusted/agit-test/pulls/1.diff","patch_url":"https://code.forgejo.org/Gusted/agit-test/pulls/1.patch","mergeable":true,"merged":false,"merged_at":null,"merge_commit_sha":null,"merged_by":null,"allow_maintainer_edit":false,"base":{"label":"main","ref":"main","sha":"79ebb873a6497c8847141ba9706b3f757196a1e6","repo_id":1414,"repo":{"id":1414,"owner":{"id":63,"login":"Gusted","login_name":"","source_id":0,"full_name":"","email":"gusted@noreply.code.forgejo.org","avatar_url":"https://code.forgejo.org/avatars/4ca5ad8bc488630869fdbd2051da61cbed7241c9c066d4e5e1dd36300f887340","html_url":"https://code.forgejo.org/Gusted","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2023-07-08T13:33:38Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":2,"following_count":0,"starred_repos_count":0,"username":"Gusted"},"name":"agit-test","full_name":"Gusted/agit-test","description":"USED FOR FORGEJO UNIT TESTING","empty":false,"private":false,"fork":false,"template":false,"parent":null,"mirror":false,"size":36,"language":"","languages_url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test/languages","html_url":"https://code.forgejo.org/Gusted/agit-test","url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test","link":"","ssh_url":"ssh://git@code.forgejo.org/Gusted/agit-test.git","clone_url":"https://code.forgejo.org/Gusted/agit-test.git","original_url":"","website":"","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":0,"open_pr_counter":1,"release_counter":0,"default_branch":"main","archived":false,"created_at":"2025-04-01T20:25:03Z","updated_at":"2025-04-01T20:25:03Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"wiki_branch":"main","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":null}},"head":{"label":"","ref":"refs/pull/1/head","sha":"667e9317ec37b977e6d3d7d43e3440636970563c","repo_id":1414,"repo":{"id":1414,"owner":{"id":63,"login":"Gusted","login_name":"","source_id":0,"full_name":"","email":"gusted@noreply.code.forgejo.org","avatar_url":"https://code.forgejo.org/avatars/4ca5ad8bc488630869fdbd2051da61cbed7241c9c066d4e5e1dd36300f887340","html_url":"https://code.forgejo.org/Gusted","language":"","is_admin":false,"last_login":"0001-01-01T00:00:00Z","created":"2023-07-08T13:33:38Z","restricted":false,"active":false,"prohibit_login":false,"location":"","pronouns":"","website":"","description":"","visibility":"public","followers_count":2,"following_count":0,"starred_repos_count":0,"username":"Gusted"},"name":"agit-test","full_name":"Gusted/agit-test","description":"USED FOR FORGEJO UNIT TESTING","empty":false,"private":false,"fork":false,"template":false,"parent":null,"mirror":false,"size":36,"language":"","languages_url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test/languages","html_url":"https://code.forgejo.org/Gusted/agit-test","url":"https://code.forgejo.org/api/v1/repos/Gusted/agit-test","link":"","ssh_url":"ssh://git@code.forgejo.org/Gusted/agit-test.git","clone_url":"https://code.forgejo.org/Gusted/agit-test.git","original_url":"","website":"","stars_count":0,"forks_count":0,"watchers_count":1,"open_issues_count":0,"open_pr_counter":1,"release_counter":0,"default_branch":"main","archived":false,"created_at":"2025-04-01T20:25:03Z","updated_at":"2025-04-01T20:25:03Z","archived_at":"1970-01-01T00:00:00Z","permissions":{"admin":true,"push":true,"pull":true},"has_issues":true,"internal_tracker":{"enable_time_tracker":true,"allow_only_contributors_to_track_time":true,"enable_issue_dependencies":true},"has_wiki":true,"wiki_branch":"main","globally_editable_wiki":false,"has_pull_requests":true,"has_projects":true,"has_releases":true,"has_packages":true,"has_actions":true,"ignore_whitespace_conflicts":false,"allow_merge_commits":true,"allow_rebase":true,"allow_rebase_explicit":true,"allow_squash_merge":true,"allow_fast_forward_only_merge":true,"allow_rebase_update":true,"default_delete_branch_after_merge":false,"default_merge_style":"merge","default_allow_maintainer_edit":false,"default_update_style":"merge","avatar_url":"","internal":false,"mirror_interval":"","object_format_name":"sha1","mirror_updated":"0001-01-01T00:00:00Z","repo_transfer":null,"topics":null}},"merge_base":"79ebb873a6497c8847141ba9706b3f757196a1e6","due_date":null,"created_at":"2025-04-01T20:28:45Z","updated_at":"2025-04-01T20:28:45Z","closed_at":null,"pin_order":0,"flow":1}] diff --git a/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fsettings%2Fapi b/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fsettings%2Fapi new file mode 100644 index 0000000000..11c4e7b8ba --- /dev/null +++ b/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fsettings%2Fapi @@ -0,0 +1,7 @@ +Content-Length: 117 +Cache-Control: max-age=0, private, must-revalidate, no-transform +Content-Type: application/json;charset=utf-8 +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN + +{"max_response_items":50,"default_paging_num":30,"default_git_trees_per_page":1000,"default_max_blob_size":10485760} diff --git a/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fversion b/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fversion new file mode 100644 index 0000000000..411ed84e24 --- /dev/null +++ b/services/migrations/testdata/code-forgejo-org/full_download/GET_%2Fapi%2Fv1%2Fversion @@ -0,0 +1,7 @@ +Cache-Control: max-age=0, private, must-revalidate, no-transform +Content-Type: application/json;charset=utf-8 +X-Content-Type-Options: nosniff +X-Frame-Options: SAMEORIGIN +Content-Length: 53 + +{"version":"11.0.0-dev-617-1d1e0ced3e+gitea-1.22.0"} diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index 400c71af1a..a16deb61a8 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -26133,6 +26133,11 @@ "format": "date-time", "x-go-name": "Deadline" }, + "flow": { + "type": "integer", + "format": "int64", + "x-go-name": "Flow" + }, "head": { "$ref": "#/definitions/PRBranchInfo" },