From 9dfdacf54f03bcfdae7ab050c17c2f8556f3a030 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 12 Jul 2025 00:39:35 +0200 Subject: [PATCH] feat: add configuration to only push mirror selected branches (#7823) Adds the ability to selectively choose which branches are pushed to a mirror. This change adds an additional text box on the repository settings for each push mirror. Existing behavior is preserved when the field is left blank. When the repository is being pushed, only branches matching the comma separated branch filter are pushed. Resolves forgejo/forgejo#7242 Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7823 Reviewed-by: Gusted Co-authored-by: Paul Campbell Co-committed-by: Paul Campbell --- models/forgejo_migrations/migrate.go | 1 + models/forgejo_migrations/v37.go | 16 + models/repo/pushmirror.go | 6 + models/repo/pushmirror_test.go | 136 ++++ modules/structs/mirror.go | 3 + options/locale_next/locale_en-US.json | 2 + routers/api/v1/repo/mirror.go | 1 + routers/web/repo/setting/setting.go | 19 + services/convert/mirror.go | 1 + services/forms/repo_form.go | 1 + services/mirror/mirror_push.go | 63 +- templates/repo/settings/options.tmpl | 6 + .../repo/settings/push_mirror_sync_modal.tmpl | 11 +- templates/swagger/v1_json.tmpl | 8 + tests/integration/api_push_mirror_test.go | 154 ++++ tests/integration/mirror_push_test.go | 671 ++++++++++++++++++ 16 files changed, 1089 insertions(+), 10 deletions(-) create mode 100644 models/forgejo_migrations/v37.go diff --git a/models/forgejo_migrations/migrate.go b/models/forgejo_migrations/migrate.go index fcea69d23f..384f382c82 100644 --- a/models/forgejo_migrations/migrate.go +++ b/models/forgejo_migrations/migrate.go @@ -111,6 +111,7 @@ var migrations = []*Migration{ NewMigration("Noop because of https://codeberg.org/forgejo/forgejo/issues/8373", NoopAddIndexToActionRunStopped), // v35 -> v36 NewMigration("Fix wiki unit default permission", FixWikiUnitDefaultPermission), + NewMigration("Add `branch_filter` to `push_mirror` table", AddPushMirrorBranchFilter), } // GetCurrentDBVersion returns the current Forgejo database version. diff --git a/models/forgejo_migrations/v37.go b/models/forgejo_migrations/v37.go new file mode 100644 index 0000000000..89358991af --- /dev/null +++ b/models/forgejo_migrations/v37.go @@ -0,0 +1,16 @@ +// Copyright 2025 The Forgejo Authors. All rights reserved. +// SPDX-License-Identifier: GPL-3.0-or-later + +package forgejo_migrations + +import ( + "xorm.io/xorm" +) + +func AddPushMirrorBranchFilter(x *xorm.Engine) error { + type PushMirror struct { + ID int64 `xorm:"pk autoincr"` + BranchFilter string `xorm:"VARCHAR(255)"` + } + return x.Sync2(new(PushMirror)) +} diff --git a/models/repo/pushmirror.go b/models/repo/pushmirror.go index d6d0d1135a..e57897fb7e 100644 --- a/models/repo/pushmirror.go +++ b/models/repo/pushmirror.go @@ -32,6 +32,7 @@ type PushMirror struct { Repo *Repository `xorm:"-"` RemoteName string RemoteAddress string `xorm:"VARCHAR(2048)"` + BranchFilter string `xorm:"VARCHAR(2048)"` // A keypair formatted in OpenSSH format. PublicKey string `xorm:"VARCHAR(100)"` @@ -122,6 +123,11 @@ func UpdatePushMirrorInterval(ctx context.Context, m *PushMirror) error { return err } +func UpdatePushMirrorBranchFilter(ctx context.Context, m *PushMirror) error { + _, err := db.GetEngine(ctx).ID(m.ID).Cols("branch_filter").Update(m) + return err +} + var DeletePushMirrors = deletePushMirrors func deletePushMirrors(ctx context.Context, opts PushMirrorOptions) error { diff --git a/models/repo/pushmirror_test.go b/models/repo/pushmirror_test.go index fbef835372..a7e063ff71 100644 --- a/models/repo/pushmirror_test.go +++ b/models/repo/pushmirror_test.go @@ -75,3 +75,139 @@ func TestPushMirrorPrivatekey(t *testing.T) { assert.Empty(t, actualPrivateKey) }) } + +func TestPushMirrorBranchFilter(t *testing.T) { + require.NoError(t, unittest.PrepareTestDatabase()) + + t.Run("Create push mirror with branch filter", func(t *testing.T) { + m := &repo_model.PushMirror{ + RepoID: 1, + RemoteName: "test-branch-filter", + BranchFilter: "main,develop", + } + unittest.AssertSuccessfulInsert(t, m) + assert.NotZero(t, m.ID) + assert.Equal(t, "main,develop", m.BranchFilter) + }) + + t.Run("Create push mirror with empty branch filter", func(t *testing.T) { + m := &repo_model.PushMirror{ + RepoID: 1, + RemoteName: "test-empty-filter", + BranchFilter: "", + } + unittest.AssertSuccessfulInsert(t, m) + assert.NotZero(t, m.ID) + assert.Empty(t, m.BranchFilter) + }) + + t.Run("Create push mirror without branch filter", func(t *testing.T) { + m := &repo_model.PushMirror{ + RepoID: 1, + RemoteName: "test-no-filter", + // BranchFilter: "", + } + unittest.AssertSuccessfulInsert(t, m) + assert.NotZero(t, m.ID) + assert.Empty(t, m.BranchFilter) + }) + + t.Run("Update branch filter", func(t *testing.T) { + m := &repo_model.PushMirror{ + RepoID: 1, + RemoteName: "test-update", + BranchFilter: "main", + } + unittest.AssertSuccessfulInsert(t, m) + + m.BranchFilter = "main,develop" + require.NoError(t, repo_model.UpdatePushMirrorBranchFilter(db.DefaultContext, m)) + + updated := unittest.AssertExistsAndLoadBean(t, &repo_model.PushMirror{ID: m.ID}) + assert.Equal(t, "main,develop", updated.BranchFilter) + }) + + t.Run("Retrieve push mirror with branch filter", func(t *testing.T) { + original := &repo_model.PushMirror{ + RepoID: 1, + RemoteName: "test-retrieve", + BranchFilter: "main,develop", + } + unittest.AssertSuccessfulInsert(t, original) + + retrieved := unittest.AssertExistsAndLoadBean(t, &repo_model.PushMirror{ID: original.ID}) + assert.Equal(t, original.BranchFilter, retrieved.BranchFilter) + assert.Equal(t, "main,develop", retrieved.BranchFilter) + }) + + t.Run("GetPushMirrorsByRepoID includes branch filter", func(t *testing.T) { + mirrors := []*repo_model.PushMirror{ + { + RepoID: 2, + RemoteName: "mirror-1", + BranchFilter: "main", + }, + { + RepoID: 2, + RemoteName: "mirror-2", + BranchFilter: "develop,feature-*", + }, + { + RepoID: 2, + RemoteName: "mirror-3", + BranchFilter: "", + }, + } + + for _, mirror := range mirrors { + unittest.AssertSuccessfulInsert(t, mirror) + } + + retrieved, count, err := repo_model.GetPushMirrorsByRepoID(db.DefaultContext, 2, db.ListOptions{}) + require.NoError(t, err) + assert.Equal(t, int64(3), count) + assert.Len(t, retrieved, 3) + + filterMap := make(map[string]string) + for _, mirror := range retrieved { + filterMap[mirror.RemoteName] = mirror.BranchFilter + } + + assert.Equal(t, "main", filterMap["mirror-1"]) + assert.Equal(t, "develop,feature-*", filterMap["mirror-2"]) + assert.Empty(t, filterMap["mirror-3"]) + }) + + t.Run("GetPushMirrorsSyncedOnCommit includes branch filter", func(t *testing.T) { + mirrors := []*repo_model.PushMirror{ + { + RepoID: 3, + RemoteName: "sync-mirror-1", + BranchFilter: "main,develop", + SyncOnCommit: true, + }, + { + RepoID: 3, + RemoteName: "sync-mirror-2", + BranchFilter: "feature-*", + SyncOnCommit: true, + }, + } + + for _, mirror := range mirrors { + unittest.AssertSuccessfulInsert(t, mirror) + } + + retrieved, err := repo_model.GetPushMirrorsSyncedOnCommit(db.DefaultContext, 3) + require.NoError(t, err) + assert.Len(t, retrieved, 2) + + filterMap := make(map[string]string) + for _, mirror := range retrieved { + filterMap[mirror.RemoteName] = mirror.BranchFilter + } + + assert.Equal(t, "main,develop", filterMap["sync-mirror-1"]) + assert.Equal(t, "feature-*", filterMap["sync-mirror-2"]) + }) +} diff --git a/modules/structs/mirror.go b/modules/structs/mirror.go index 1b6566803a..4909ae20ca 100644 --- a/modules/structs/mirror.go +++ b/modules/structs/mirror.go @@ -13,6 +13,7 @@ type CreatePushMirrorOption struct { Interval string `json:"interval"` SyncOnCommit bool `json:"sync_on_commit"` UseSSH bool `json:"use_ssh"` + BranchFilter string `json:"branch_filter"` } // PushMirror represents information of a push mirror @@ -29,4 +30,6 @@ type PushMirror struct { Interval string `json:"interval"` SyncOnCommit bool `json:"sync_on_commit"` PublicKey string `json:"public_key"` + + BranchFilter string `json:"branch_filter"` } diff --git a/options/locale_next/locale_en-US.json b/options/locale_next/locale_en-US.json index e08c8b2aee..1778a1fc6c 100644 --- a/options/locale_next/locale_en-US.json +++ b/options/locale_next/locale_en-US.json @@ -55,6 +55,8 @@ "repo.form.cannot_create": "All spaces in which you can create repositories have reached the limit of repositories.", "repo.issue_indexer.title": "Issue Indexer", "search.milestone_kind": "Search milestones…", + "repo.settings.push_mirror.branch_filter.label": "Branch filter (optional)", + "repo.settings.push_mirror.branch_filter.description": "Branches to be mirrored. Leave blank to mirror all branches. See %[2]s documentation for syntax. Examples: main, release/*", "incorrect_root_url": "This Forgejo instance is configured to be served on \"%s\". You are currently viewing Forgejo through a different URL, which may cause parts of the application to break. The canonical URL is controlled by Forgejo admins via the ROOT_URL setting in the app.ini.", "themes.names.forgejo-auto": "Forgejo (follow system theme)", "themes.names.forgejo-light": "Forgejo light", diff --git a/routers/api/v1/repo/mirror.go b/routers/api/v1/repo/mirror.go index bc48c6acb7..f08867dee4 100644 --- a/routers/api/v1/repo/mirror.go +++ b/routers/api/v1/repo/mirror.go @@ -389,6 +389,7 @@ func CreatePushMirror(ctx *context.APIContext, mirrorOption *api.CreatePushMirro Interval: interval, SyncOnCommit: mirrorOption.SyncOnCommit, RemoteAddress: remoteAddress, + BranchFilter: mirrorOption.BranchFilter, } var plainPrivateKey []byte diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index 6f35e19880..595fdace83 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -6,6 +6,7 @@ package setting import ( + go_context "context" "errors" "fmt" "net/http" @@ -589,6 +590,23 @@ func SettingsPost(ctx *context.Context) { ctx.ServerError("UpdatePushMirrorInterval", err) return } + + if m.BranchFilter != form.PushMirrorBranchFilter { + // replace `remote..push` in config and db + m.BranchFilter = form.PushMirrorBranchFilter + if err := db.WithTx(ctx, func(ctx go_context.Context) error { + // Update the DB + if err = repo_model.UpdatePushMirrorBranchFilter(ctx, m); err != nil { + return err + } + // Update the repo config + return mirror_service.UpdatePushMirrorBranchFilter(ctx, m) + }); err != nil { + ctx.ServerError("UpdatePushMirrorBranchFilter", err) + return + } + } + // Background why we are adding it to Queue // If we observed its implementation in the context of `push-mirror-sync` where it // is evident that pushing to the queue is necessary for updates. @@ -684,6 +702,7 @@ func SettingsPost(ctx *context.Context) { SyncOnCommit: form.PushMirrorSyncOnCommit, Interval: interval, RemoteAddress: remoteAddress, + BranchFilter: form.PushMirrorBranchFilter, } var plainPrivateKey []byte diff --git a/services/convert/mirror.go b/services/convert/mirror.go index 9e7d2659ab..5a815f3a5c 100644 --- a/services/convert/mirror.go +++ b/services/convert/mirror.go @@ -23,5 +23,6 @@ func ToPushMirror(ctx context.Context, pm *repo_model.PushMirror) (*api.PushMirr Interval: pm.Interval.String(), SyncOnCommit: pm.SyncOnCommit, PublicKey: pm.GetPublicKey(), + BranchFilter: pm.BranchFilter, }, nil } diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index bb81e939b0..d040b41395 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -141,6 +141,7 @@ type RepoSettingForm struct { PushMirrorSyncOnCommit bool PushMirrorInterval string PushMirrorUseSSH bool + PushMirrorBranchFilter string `binding:"MaxSize(2048)" preprocess:"TrimSpace"` Private bool Template bool EnablePrune bool diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go index 11b8ad459a..fdd02dedea 100644 --- a/services/mirror/mirror_push.go +++ b/services/mirror/mirror_push.go @@ -33,19 +33,22 @@ var AddPushMirrorRemote = addPushMirrorRemote func addPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr string) error { addRemoteAndConfig := func(addr, path string) error { - cmd := git.NewCommand(ctx, "remote", "add", "--mirror=push").AddDynamicArguments(m.RemoteName, addr) - if strings.Contains(addr, "://") && strings.Contains(addr, "@") { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, util.SanitizeCredentialURLs(addr), path)) + var cmd *git.Command + if m.BranchFilter == "" { + cmd = git.NewCommand(ctx, "remote", "add", "--mirror").AddDynamicArguments(m.RemoteName, addr) } else { - cmd.SetDescription(fmt.Sprintf("remote add %s --mirror=push %s [repo_path: %s]", m.RemoteName, addr, path)) + cmd = git.NewCommand(ctx, "remote", "add").AddDynamicArguments(m.RemoteName, addr) + } + if strings.Contains(addr, "://") && strings.Contains(addr, "@") { + cmd.SetDescription(fmt.Sprintf("remote add %s %s [repo_path: %s]", m.RemoteName, util.SanitizeCredentialURLs(addr), path)) + } else { + cmd.SetDescription(fmt.Sprintf("remote add %s %s [repo_path: %s]", m.RemoteName, addr, path)) } if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil { return err } - if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { - return err - } - if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + err := addRemotePushRefSpecs(ctx, path, m) + if err != nil { return err } return nil @@ -67,6 +70,49 @@ func addPushMirrorRemote(ctx context.Context, m *repo_model.PushMirror, addr str return nil } +func addRemotePushRefSpecs(ctx context.Context, path string, m *repo_model.PushMirror) error { + if m.BranchFilter == "" { + // If there is no branch filter, set the push refspecs to mirror all branches and tags. + if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/heads/*:refs/heads/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + } else { + branches := strings.SplitSeq(m.BranchFilter, ",") + for branch := range branches { + branch = strings.TrimSpace(branch) + if branch == "" { + continue + } + refspec := fmt.Sprintf("+refs/heads/%s:refs/heads/%s", branch, branch) + if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", refspec).RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + } + } + if _, _, err := git.NewCommand(ctx, "config", "--add").AddDynamicArguments("remote."+m.RemoteName+".push", "+refs/tags/*:refs/tags/*").RunStdString(&git.RunOpts{Dir: path}); err != nil { + return err + } + return nil +} + +func UpdatePushMirrorBranchFilter(ctx context.Context, m *repo_model.PushMirror) error { + path := m.Repo.RepoPath() + + // First, remove all existing push refspecs for this remote + cmd := git.NewCommand(ctx, "config", "--unset-all").AddDynamicArguments("remote." + m.RemoteName + ".push") + if _, _, err := cmd.RunStdString(&git.RunOpts{Dir: path}); err != nil { + // Ignore error if the key doesn't exist + if !strings.Contains(err.Error(), "does not exist") { + return err + } + } + err := addRemotePushRefSpecs(ctx, path, m) + if err != nil { + return err + } + return nil +} + // RemovePushMirrorRemote removes the push mirror remote. func RemovePushMirrorRemote(ctx context.Context, m *repo_model.PushMirror) error { cmd := git.NewCommand(ctx, "remote", "rm").AddDynamicArguments(m.RemoteName) @@ -212,7 +258,6 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error { return util.SanitizeErrorCredentialURLs(err) } - return nil } diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 797dbe403b..c8061a75b0 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -254,6 +254,7 @@ data-modal-push-mirror-edit-id="{{.ID}}" data-modal-push-mirror-edit-interval="{{.Interval}}" data-modal-push-mirror-edit-address="{{.RemoteAddress}}" + data-modal-push-mirror-edit-branch-filter="{{.BranchFilter}}" > {{svg "octicon-pencil" 14}} @@ -288,6 +289,11 @@

{{ctx.Locale.Tr "repo.mirror_address_desc"}}

+
+ + +

{{ctx.Locale.Tr "repo.settings.push_mirror.branch_filter.description" "https://forgejo.org/docs/latest/user/repo-mirror/#branch-filter" "forgejo"}}

+
{{ctx.Locale.Tr "repo.need_auth"}} diff --git a/templates/repo/settings/push_mirror_sync_modal.tmpl b/templates/repo/settings/push_mirror_sync_modal.tmpl index e8dad61a48..32f0994f03 100644 --- a/templates/repo/settings/push_mirror_sync_modal.tmpl +++ b/templates/repo/settings/push_mirror_sync_modal.tmpl @@ -15,7 +15,16 @@
- +
+ +
+
+
+ +
+ +
+

{{ctx.Locale.Tr "repo.settings.push_mirror.branch_filter.description" "https://forgejo.org/docs/latest/user/repo-mirror/#branch-filter" "forgejo"}}